From 951af38fe7133f1e4b7e2481a7d0a2c7d535f2c3 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Tue, 8 Jun 2021 00:44:25 +0000 Subject: [PATCH] SWDEV-290076 - Remove backend implementation from HIP common Updated cmake to use HIP_AMD_BACKEND_SOURCE_DIR Change-Id: Ibeaae0d75ffa49a20752a55506e25dbfbe3b5b68 --- CMakeLists.txt | 13 +- packaging/hip-nvcc.txt | 1 + packaging/hip-rocclr.txt | 1 + src/hipamd/CMakeLists.txt | 286 - src/hipamd/cmake/FindROCclr.cmake | 51 - .../hip/amd_detail/amd_channel_descriptor.h | 348 - .../hip/amd_detail/amd_device_functions.h | 1369 -- .../include/hip/amd_detail/amd_driver_types.h | 478 - .../include/hip/amd_detail/amd_hip_atomic.h | 691 - .../include/hip/amd_detail/amd_hip_common.h | 32 - .../include/hip/amd_detail/amd_hip_complex.h | 309 - .../amd_detail/amd_hip_cooperative_groups.h | 510 - .../include/hip/amd_detail/amd_hip_fp16.h | 1663 --- .../include/hip/amd_detail/amd_hip_runtime.h | 417 - .../hip/amd_detail/amd_hip_surface_types.h | 54 - .../hip/amd_detail/amd_hip_texture_types.h | 97 - .../hip/amd_detail/amd_hip_vector_types.h | 1598 --- .../include/hip/amd_detail/amd_hiprtc.h | 94 - .../hip/amd_detail/amd_library_types.h | 41 - .../hip/amd_detail/amd_math_functions.h | 1502 --- .../hip/amd_detail/amd_surface_functions.h | 59 - .../hip/amd_detail/amd_texture_types.h | 109 - .../include/hip/amd_detail/concepts.hpp | 30 - .../hip/amd_detail/device_library_decls.h | 118 - .../hip/amd_detail/functional_grid_launch.hpp | 218 - .../include/hip/amd_detail/grid_launch.h | 67 - .../include/hip/amd_detail/grid_launch.hpp | 50 - .../hip/amd_detail/grid_launch_GGL.hpp | 26 - src/hipamd/include/hip/amd_detail/helpers.hpp | 137 - .../hip_cooperative_groups_helper.h | 179 - .../include/hip/amd_detail/hip_fp16_gcc.h | 254 - .../hip/amd_detail/hip_fp16_math_fwd.h | 86 - src/hipamd/include/hip/amd_detail/hip_ldg.h | 100 - .../include/hip/amd_detail/hip_memory.h | 114 - .../include/hip/amd_detail/hip_runtime_prof.h | 77 - .../include/hip/amd_detail/host_defines.h | 72 - .../include/hip/amd_detail/hsa_helpers.hpp | 102 - .../include/hip/amd_detail/llvm_intrinsics.h | 41 - .../amd_detail/macro_based_grid_launch.hpp | 798 -- src/hipamd/include/hip/amd_detail/math_fwd.h | 714 - .../include/hip/amd_detail/ockl_image.h | 135 - .../include/hip/amd_detail/program_state.hpp | 107 - .../hip/amd_detail/texture_fetch_functions.h | 388 - .../hip/amd_detail/texture_functions.h | 11102 ---------------- .../amd_detail/texture_indirect_functions.h | 503 - src/hipamd/include/hip/hcc_detail | 1 - src/hipamd/include/hip/nvcc_detail | 1 - .../nvidia_detail/nvidia_channel_descriptor.h | 28 - .../hip/nvidia_detail/nvidia_hip_complex.h | 119 - .../nvidia_hip_cooperative_groups.h | 12 - .../hip/nvidia_detail/nvidia_hip_runtime.h | 122 - .../nvidia_detail/nvidia_hip_runtime_api.h | 2195 --- .../nvidia_detail/nvidia_hip_texture_types.h | 6 - .../include/hip/nvidia_detail/nvidia_hiprtc.h | 168 - src/hipamd/src/amd_hsa_elf.hpp | 118 - src/hipamd/src/amdhip.def | 293 - src/hipamd/src/cl_d3d10.cpp | 1451 -- src/hipamd/src/cl_d3d11.cpp | 1571 --- src/hipamd/src/cl_d3d9.cpp | 855 -- src/hipamd/src/cl_gl.cpp | 2432 ---- src/hipamd/src/cl_gl_amd.hpp | 379 - src/hipamd/src/cl_lqdflash_amd.cpp | 312 - src/hipamd/src/cl_lqdflash_amd.h | 58 - src/hipamd/src/fixme.cpp | 32 - src/hipamd/src/hip_activity.cpp | 35 - src/hipamd/src/hip_code_object.cpp | 850 -- src/hipamd/src/hip_code_object.hpp | 165 - src/hipamd/src/hip_context.cpp | 377 - src/hipamd/src/hip_conversions.hpp | 903 -- src/hipamd/src/hip_device.cpp | 240 - src/hipamd/src/hip_device_runtime.cpp | 563 - src/hipamd/src/hip_error.cpp | 176 - src/hipamd/src/hip_event.cpp | 431 - src/hipamd/src/hip_event.hpp | 151 - src/hipamd/src/hip_fatbin.cpp | 158 - src/hipamd/src/hip_fatbin.hpp | 87 - src/hipamd/src/hip_formatting.hpp | 853 -- src/hipamd/src/hip_global.cpp | 196 - src/hipamd/src/hip_global.hpp | 127 - src/hipamd/src/hip_graph.cpp | 415 - src/hipamd/src/hip_graph_capture.hpp | 48 - src/hipamd/src/hip_graph_helper.hpp | 35 - src/hipamd/src/hip_graph_internal.cpp | 364 - src/hipamd/src/hip_graph_internal.hpp | 355 - src/hipamd/src/hip_hcc.def.in | 293 - src/hipamd/src/hip_hcc.map.in | 310 - src/hipamd/src/hip_hcc.rc | 75 - src/hipamd/src/hip_hmm.cpp | 224 - src/hipamd/src/hip_intercept.cpp | 81 - src/hipamd/src/hip_internal.hpp | 353 - src/hipamd/src/hip_memory.cpp | 2852 ---- src/hipamd/src/hip_module.cpp | 675 - src/hipamd/src/hip_peer.cpp | 250 - src/hipamd/src/hip_platform.cpp | 945 -- src/hipamd/src/hip_platform.hpp | 97 - src/hipamd/src/hip_prof_api.h | 270 - src/hipamd/src/hip_prof_gen.py | 673 - src/hipamd/src/hip_profile.cpp | 40 - src/hipamd/src/hip_rtc.cpp | 419 - src/hipamd/src/hip_stream.cpp | 598 - src/hipamd/src/hip_stream_ops.cpp | 129 - src/hipamd/src/hip_surface.cpp | 37 - src/hipamd/src/hip_texture.cpp | 1303 -- src/hipamd/src/hiprtc_internal.hpp | 65 - src/hipamd/src/trace_helper.h | 246 - 105 files changed, 12 insertions(+), 51746 deletions(-) delete mode 100755 src/hipamd/CMakeLists.txt delete mode 100644 src/hipamd/cmake/FindROCclr.cmake delete mode 100644 src/hipamd/include/hip/amd_detail/amd_channel_descriptor.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_device_functions.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_driver_types.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_atomic.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_common.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_complex.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_fp16.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_runtime.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_surface_types.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_texture_types.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hip_vector_types.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_hiprtc.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_library_types.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_math_functions.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_surface_functions.h delete mode 100644 src/hipamd/include/hip/amd_detail/amd_texture_types.h delete mode 100644 src/hipamd/include/hip/amd_detail/concepts.hpp delete mode 100644 src/hipamd/include/hip/amd_detail/device_library_decls.h delete mode 100644 src/hipamd/include/hip/amd_detail/functional_grid_launch.hpp delete mode 100644 src/hipamd/include/hip/amd_detail/grid_launch.h delete mode 100644 src/hipamd/include/hip/amd_detail/grid_launch.hpp delete mode 100644 src/hipamd/include/hip/amd_detail/grid_launch_GGL.hpp delete mode 100644 src/hipamd/include/hip/amd_detail/helpers.hpp delete mode 100644 src/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h delete mode 100644 src/hipamd/include/hip/amd_detail/hip_fp16_gcc.h delete mode 100644 src/hipamd/include/hip/amd_detail/hip_fp16_math_fwd.h delete mode 100644 src/hipamd/include/hip/amd_detail/hip_ldg.h delete mode 100644 src/hipamd/include/hip/amd_detail/hip_memory.h delete mode 100644 src/hipamd/include/hip/amd_detail/hip_runtime_prof.h delete mode 100644 src/hipamd/include/hip/amd_detail/host_defines.h delete mode 100644 src/hipamd/include/hip/amd_detail/hsa_helpers.hpp delete mode 100644 src/hipamd/include/hip/amd_detail/llvm_intrinsics.h delete mode 100644 src/hipamd/include/hip/amd_detail/macro_based_grid_launch.hpp delete mode 100644 src/hipamd/include/hip/amd_detail/math_fwd.h delete mode 100644 src/hipamd/include/hip/amd_detail/ockl_image.h delete mode 100644 src/hipamd/include/hip/amd_detail/program_state.hpp delete mode 100644 src/hipamd/include/hip/amd_detail/texture_fetch_functions.h delete mode 100644 src/hipamd/include/hip/amd_detail/texture_functions.h delete mode 100644 src/hipamd/include/hip/amd_detail/texture_indirect_functions.h delete mode 120000 src/hipamd/include/hip/hcc_detail delete mode 120000 src/hipamd/include/hip/nvcc_detail delete mode 100644 src/hipamd/include/hip/nvidia_detail/nvidia_channel_descriptor.h delete mode 100644 src/hipamd/include/hip/nvidia_detail/nvidia_hip_complex.h delete mode 100644 src/hipamd/include/hip/nvidia_detail/nvidia_hip_cooperative_groups.h delete mode 100644 src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime.h delete mode 100644 src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime_api.h delete mode 100644 src/hipamd/include/hip/nvidia_detail/nvidia_hip_texture_types.h delete mode 100644 src/hipamd/include/hip/nvidia_detail/nvidia_hiprtc.h delete mode 100644 src/hipamd/src/amd_hsa_elf.hpp delete mode 100644 src/hipamd/src/amdhip.def delete mode 100644 src/hipamd/src/cl_d3d10.cpp delete mode 100644 src/hipamd/src/cl_d3d11.cpp delete mode 100644 src/hipamd/src/cl_d3d9.cpp delete mode 100644 src/hipamd/src/cl_gl.cpp delete mode 100644 src/hipamd/src/cl_gl_amd.hpp delete mode 100644 src/hipamd/src/cl_lqdflash_amd.cpp delete mode 100644 src/hipamd/src/cl_lqdflash_amd.h delete mode 100644 src/hipamd/src/fixme.cpp delete mode 100644 src/hipamd/src/hip_activity.cpp delete mode 100755 src/hipamd/src/hip_code_object.cpp delete mode 100755 src/hipamd/src/hip_code_object.hpp delete mode 100755 src/hipamd/src/hip_context.cpp delete mode 100644 src/hipamd/src/hip_conversions.hpp delete mode 100644 src/hipamd/src/hip_device.cpp delete mode 100755 src/hipamd/src/hip_device_runtime.cpp delete mode 100644 src/hipamd/src/hip_error.cpp delete mode 100755 src/hipamd/src/hip_event.cpp delete mode 100644 src/hipamd/src/hip_event.hpp delete mode 100755 src/hipamd/src/hip_fatbin.cpp delete mode 100755 src/hipamd/src/hip_fatbin.hpp delete mode 100644 src/hipamd/src/hip_formatting.hpp delete mode 100755 src/hipamd/src/hip_global.cpp delete mode 100755 src/hipamd/src/hip_global.hpp delete mode 100644 src/hipamd/src/hip_graph.cpp delete mode 100644 src/hipamd/src/hip_graph_capture.hpp delete mode 100644 src/hipamd/src/hip_graph_helper.hpp delete mode 100644 src/hipamd/src/hip_graph_internal.cpp delete mode 100644 src/hipamd/src/hip_graph_internal.hpp delete mode 100755 src/hipamd/src/hip_hcc.def.in delete mode 100755 src/hipamd/src/hip_hcc.map.in delete mode 100644 src/hipamd/src/hip_hcc.rc delete mode 100644 src/hipamd/src/hip_hmm.cpp delete mode 100755 src/hipamd/src/hip_intercept.cpp delete mode 100755 src/hipamd/src/hip_internal.hpp delete mode 100755 src/hipamd/src/hip_memory.cpp delete mode 100755 src/hipamd/src/hip_module.cpp delete mode 100755 src/hipamd/src/hip_peer.cpp delete mode 100755 src/hipamd/src/hip_platform.cpp delete mode 100755 src/hipamd/src/hip_platform.hpp delete mode 100644 src/hipamd/src/hip_prof_api.h delete mode 100755 src/hipamd/src/hip_prof_gen.py delete mode 100644 src/hipamd/src/hip_profile.cpp delete mode 100755 src/hipamd/src/hip_rtc.cpp delete mode 100755 src/hipamd/src/hip_stream.cpp delete mode 100644 src/hipamd/src/hip_stream_ops.cpp delete mode 100644 src/hipamd/src/hip_surface.cpp delete mode 100755 src/hipamd/src/hip_texture.cpp delete mode 100644 src/hipamd/src/hiprtc_internal.hpp delete mode 100644 src/hipamd/src/trace_helper.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9153f08d93..b573a9a79a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,12 @@ else() set(_pchStatus 0) endif() +# With HIP project split plan, for interim puropose it is REQUIRED to pass +# HIP_AMD_BACKEND_SOURCE_DIR= as cmake variable_ +if(NOT DEFINED HIP_AMD_BACKEND_SOURCE_DIR) + message(FATAL_ERROR "HIP_AMD_BACKEND_SOURCE_DIR not defined ") +endif() + ############################# # Setup config generation ############################# @@ -285,7 +291,8 @@ set(_versionInfoHeader file(WRITE "${PROJECT_BINARY_DIR}/include/hip/hip_version.h" ${_versionInfoHeader}) if(HIP_RUNTIME STREQUAL "rocclr") - add_subdirectory(src/hipamd) + set(HIP_COMMON_DIR ${PROJECT_SOURCE_DIR}) + add_subdirectory(${HIP_AMD_BACKEND_SOURCE_DIR} src/hipamd) endif() # Generate .hipInfo @@ -322,7 +329,7 @@ if(NOT ${INSTALL_SOURCE} EQUAL 0) install(CODE "file(REMOVE_RECURSE ${CMAKE_INSTALL_PREFIX}/include)") install(DIRECTORY include DESTINATION .) - install(DIRECTORY src/hipamd/include/hip/ DESTINATION include/hip/) + install(DIRECTORY ${HIP_AMD_BACKEND_SOURCE_DIR}/include/hip/ DESTINATION include/hip/) install(DIRECTORY cmake DESTINATION .) endif() @@ -479,7 +486,7 @@ set(HIP_SRC_PATH ${CMAKE_CURRENT_SOURCE_DIR}) if(HIP_PLATFORM STREQUAL "nvidia") execute_process(COMMAND "${CMAKE_COMMAND}" -E copy_directory "${HIP_SRC_PATH}/include" "${HIP_ROOT_DIR}/include" RESULT_VARIABLE RUN_HIT ERROR_QUIET) endif() -execute_process(COMMAND "${CMAKE_COMMAND}" -E copy_directory "${HIP_SRC_PATH}/src/hipamd/include/hip/" "${HIP_ROOT_DIR}/include/hip/" RESULT_VARIABLE RUN_HIT ERROR_QUIET) +execute_process(COMMAND "${CMAKE_COMMAND}" -E copy_directory "${HIP_AMD_BACKEND_SOURCE_DIR}/include/hip/" "${HIP_ROOT_DIR}/include/hip/" RESULT_VARIABLE RUN_HIT ERROR_QUIET) execute_process(COMMAND "${CMAKE_COMMAND}" -E copy_directory "${HIP_SRC_PATH}/cmake" "${HIP_ROOT_DIR}/cmake" RESULT_VARIABLE RUN_HIT ERROR_QUIET) if(${RUN_HIT} EQUAL 0) execute_process(COMMAND "${CMAKE_COMMAND}" -E copy_directory "${HIP_SRC_PATH}/bin" "${HIP_ROOT_DIR}/bin" RESULT_VARIABLE RUN_HIT ERROR_QUIET) diff --git a/packaging/hip-nvcc.txt b/packaging/hip-nvcc.txt index 88e0a8ffb4..db7eaf0988 100644 --- a/packaging/hip-nvcc.txt +++ b/packaging/hip-nvcc.txt @@ -20,6 +20,7 @@ cmake_minimum_required(VERSION 2.8.3) project(hip_nvcc) +install(DIRECTORY @PROJECT_BINARY_DIR@/include/hip/nvcc_detail DESTINATION .) ############################# # Packaging steps diff --git a/packaging/hip-rocclr.txt b/packaging/hip-rocclr.txt index f2a65034ed..e4c3e447f2 100644 --- a/packaging/hip-rocclr.txt +++ b/packaging/hip-rocclr.txt @@ -29,6 +29,7 @@ else() install(FILES @PROJECT_BINARY_DIR@/lib/libamdhip64.a DESTINATION lib) endif() +install(DIRECTORY @PROJECT_BINARY_DIR@/include/hip/amd_detail DESTINATION .) install(FILES @PROJECT_BINARY_DIR@/.hipInfo DESTINATION lib) install(FILES @PROJECT_BINARY_DIR@/hip-config.cmake @PROJECT_BINARY_DIR@/hip-config-version.cmake DESTINATION lib/cmake/hip) install(FILES @PROJECT_BINARY_DIR@/src/hipamd/hip-lang-config.cmake @PROJECT_BINARY_DIR@/src/hipamd/hip-lang-config-version.cmake DESTINATION lib/cmake/hip-lang) diff --git a/src/hipamd/CMakeLists.txt b/src/hipamd/CMakeLists.txt deleted file mode 100755 index 4982fb93f6..0000000000 --- a/src/hipamd/CMakeLists.txt +++ /dev/null @@ -1,286 +0,0 @@ -# Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -cmake_minimum_required(VERSION 3.5.1) - -include(GNUInstallDirs) - -if(ADDRESS_SANITIZER) - set(ASAN_LINKER_FLAGS "-fsanitize=address") - set(ASAN_COMPILER_FLAGS "-fno-omit-frame-pointer -fsanitize=address") - - if(NOT CMAKE_COMPILER_IS_GNUCC) - if(BUILD_SHARED_LIBS) - set(ASAN_LINKER_FLAGS "${ASAN_LINKER_FLAGS} -shared-libsan") - else() - set(ASAN_LINKER_FLAGS "${ASAN_LINKER_FLAGS} -static-libsan") - endif() - endif() - - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ASAN_COMPILER_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ASAN_COMPILER_FLAGS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ASAN_LINKER_FLAGS} -s") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${ASAN_LINKER_FLAGS}") -endif() - -option(BUILD_SHARED_LIBS "Build the shared library" ON) - -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") -find_package(ROCclr) - -if(BUILD_SHARED_LIBS) - add_library(amdhip64 SHARED) - # Windows doesn't have a strip utility, so CMAKE_STRIP won't be set. - if((CMAKE_BUILD_TYPE STREQUAL "Release") AND NOT ("${CMAKE_STRIP}" STREQUAL "")) - add_custom_command(TARGET amdhip64 POST_BUILD COMMAND ${CMAKE_STRIP} $) - endif() -else() - add_library(amdhip64 STATIC $) -endif() - -set_target_properties(amdhip64 PROPERTIES - CXX_STANDARD 14 - CXX_STANDARD_REQUIRED ON - CXX_EXTENSIONS OFF - POSITION_INDEPENDENT_CODE ON - # Workaround for many places in the HIP project - # having hardcoded references to build/lib/libamdhip64.so - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) - -if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set_target_properties(amdhip64 PROPERTIES OUTPUT_NAME "amdhip64") -else() - set_target_properties(amdhip64 PROPERTIES OUTPUT_NAME "amdhip32") -endif() - -# Disable versioning for Windows -# as currently HIP_LIB_VERSION_STRING and HIP_LIB_VERSION_MAJOR -# are not being populated -if(NOT WIN32) - if(BUILD_SHARED_LIBS) - set_target_properties(amdhip64 PROPERTIES - VERSION ${HIP_LIB_VERSION_STRING} - SOVERSION ${HIP_LIB_VERSION_MAJOR}) - endif() -endif() - -target_sources(amdhip64 PRIVATE - src/cl_gl.cpp - src/cl_lqdflash_amd.cpp - src/fixme.cpp - src/hip_activity.cpp - src/hip_code_object.cpp - src/hip_context.cpp - src/hip_device_runtime.cpp - src/hip_device.cpp - src/hip_error.cpp - src/hip_event.cpp - src/hip_fatbin.cpp - src/hip_global.cpp - src/hip_graph_internal.cpp - src/hip_graph.cpp - src/hip_hmm.cpp - src/hip_intercept.cpp - src/hip_memory.cpp - src/hip_module.cpp - src/hip_peer.cpp - src/hip_platform.cpp - src/hip_profile.cpp - src/hip_rtc.cpp - src/hip_stream_ops.cpp - src/hip_stream.cpp - src/hip_surface.cpp - src/hip_texture.cpp) - -if(WIN32) - target_sources(amdhip64 PRIVATE - src/cl_d3d9.cpp - src/cl_d3d10.cpp - src/cl_d3d11.cpp) -endif() - -if(BUILD_SHARED_LIBS) - if(WIN32) - target_sources(amdhip64 PRIVATE src/amdhip.def) - else() - target_link_libraries(amdhip64 PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_LIST_DIR}/src/hip_hcc.map.in") - set_target_properties(amdhip64 PROPERTIES LINK_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/src/hip_hcc.map.in") - endif() -endif() - -target_include_directories(amdhip64 - PRIVATE - ${PROJECT_SOURCE_DIR}/src/hipamd/include - ${PROJECT_SOURCE_DIR}/include - ${PROJECT_BINARY_DIR}/include) - -target_compile_definitions(amdhip64 PRIVATE __HIP_PLATFORM_AMD__) - -target_link_libraries(amdhip64 PRIVATE ${CMAKE_DL_LIBS}) -# Additional dependencies for hipRTC -if(WIN32) - target_link_libraries(amdhip64 PRIVATE Dbghelp.lib) -endif() - -# Note in static case we cannot link against rocclr. -# If we would, we'd also have to export rocclr and have hipcc pass it to the linker. -if(BUILD_SHARED_LIBS) - target_link_libraries(amdhip64 PRIVATE rocclr) -else() - target_compile_definitions(amdhip64 PRIVATE $) - target_include_directories(amdhip64 PRIVATE $) -endif() - -# Short-Term solution for pre-compiled headers for online compilation -# Enable pre compiled header -if(__HIP_ENABLE_PCH) - find_package(LLVM REQUIRED CONFIG - PATHS - /opt/rocm/llvm) - # find_package(LLVM) returns the lib/cmake/llvm location. We require the root. - set(HIP_LLVM_ROOT "${LLVM_DIR}/../../..") - - # execute_process(COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/../bin/hip_embed_pch.sh ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/include ${HIP_LLVM_ROOT}" COMMAND_ECHO STDERR RESULT_VARIABLE EMBED_PCH_RC) - execute_process(COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/../../bin/hip_embed_pch.sh ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/src/hipamd/include ${HIP_LLVM_ROOT}" COMMAND_ECHO STDERR RESULT_VARIABLE EMBED_PCH_RC) - if (EMBED_PCH_RC AND NOT EMBED_PCH_RC EQUAL 0) - message(FATAL_ERROR "Failed to embed PCH") - endif() - - target_compile_definitions(amdhip64 PRIVATE __HIP_ENABLE_PCH) - target_sources(amdhip64 PRIVATE ${CMAKE_BINARY_DIR}/hip_pch.o) -endif() - -# Enable preprocessed hiprtc-builtins library -if(__HIP_ENABLE_RTC) - find_package(LLVM REQUIRED CONFIG - PATHS - /opt/rocm/llvm) - # find_package(LLVM) returns the lib/cmake/llvm location. We require the root. - set(HIP_LLVM_ROOT "${LLVM_DIR}/../../..") - - if(WIN32) - set(HIPRTC_LIB_NAME "hiprtc-builtins64_${HIP_LIB_VERSION_MAJOR}${HIP_LIB_VERSION_MINOR}.dll") - else() - set(HIPRTC_LIB_NAME "libhiprtc-builtins.so.${HIP_LIB_VERSION_MAJOR}.${HIP_LIB_VERSION_MINOR}") - endif() - execute_process( - COMMAND sh -c "mkdir -p ${PROJECT_BINARY_DIR}/lib; ${CMAKE_CURRENT_SOURCE_DIR}/../../bin/hip_embed_pch.sh ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/src/hipamd/include ${HIP_LLVM_ROOT} -r ${PROJECT_BINARY_DIR}/lib/${HIPRTC_LIB_NAME}" - COMMAND_ECHO STDERR - RESULT_VARIABLE EMBED_RTC_RC - ) - if (EMBED_RTC_RC AND NOT EMBED_RTC_RC EQUAL 0) - message(FATAL_ERROR "Failed to create hiprtc shared lib") - endif() - install(FILES ${PROJECT_BINARY_DIR}/lib/${HIPRTC_LIB_NAME} DESTINATION lib) -endif() - -############################# -# Profiling API support -############################# -# Generate profiling API macros/structures header -set(PROF_API_STR "${PROJECT_BINARY_DIR}/include/hip/amd_detail/hip_prof_str.h") -set(PROF_API_HDR "${PROJECT_SOURCE_DIR}/include/hip/hip_runtime_api.h") -set(PROF_API_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src") -set(PROF_API_GEN "${CMAKE_CURRENT_SOURCE_DIR}/src/hip_prof_gen.py") -set(PROF_API_LOG "${PROJECT_BINARY_DIR}/hip_prof_gen.log.txt") - -find_package(PythonInterp REQUIRED) -add_custom_command(OUTPUT ${PROF_API_STR} - COMMAND ${PYTHON_EXECUTABLE} ${PROF_API_GEN} -v -t --priv ${OPT_PROF_API} ${PROF_API_HDR} ${PROF_API_SRC} ${PROF_API_STR} - OUTPUT_FILE ${PROF_API_LOG} - DEPENDS ${PROF_API_HDR} ${PROF_API_GEN} - COMMENT "Generating profiling primitives: ${PROF_API_STR}") - -add_custom_target(gen-prof-api-str-header ALL - DEPENDS ${PROF_API_STR} - SOURCES ${PROF_API_HDR}) - -set_target_properties(amdhip64 PROPERTIES PUBLIC_HEADER ${PROF_API_STR}) - -option(USE_PROF_API ON "Enable roctracer integration") -# Enable profiling API -if(USE_PROF_API) - find_path(PROF_API_HEADER_DIR prof_protocol.h - HINTS - ${PROF_API_HEADER_PATH} - PATHS - ${ROCM_PATH}/roctracer - PATH_SUFFIXES - include/ext) - - if(NOT PROF_API_HEADER_DIR) - message(WARNING "Profiling API header not found. Disabling roctracer integration. Use -DPROF_API_HEADER_PATH=") - else() - target_compile_definitions(amdhip64 PUBLIC USE_PROF_API=1) - target_include_directories(amdhip64 PUBLIC ${PROF_API_HEADER_DIR}) - message(STATUS "Profiling API: ${PROF_API_HEADER_DIR}") - endif() -endif() - -add_dependencies(amdhip64 gen-prof-api-str-header) - -add_custom_command(TARGET amdhip64 POST_BUILD COMMAND - ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/.hipInfo ${PROJECT_BINARY_DIR}/lib/.hipInfo) -add_custom_command(TARGET amdhip64 POST_BUILD COMMAND - ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/include ${PROJECT_BINARY_DIR}/include) - -add_library(host INTERFACE) -target_link_libraries(host INTERFACE amdhip64) - -add_library(device INTERFACE) -target_link_libraries(device INTERFACE host) - -INSTALL(TARGETS amdhip64 host device - EXPORT hip-targets - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -INSTALL(EXPORT hip-targets DESTINATION ${CONFIG_PACKAGE_INSTALL_DIR} NAMESPACE hip::) - -INSTALL(TARGETS amdhip64 host device - EXPORT hip-lang-targets - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -INSTALL(EXPORT hip-lang-targets DESTINATION ${CONFIG_LANG_PACKAGE_INSTALL_DIR} NAMESPACE hip-lang::) - -if(NOT WIN32) -include(CMakePackageConfigHelpers) - -configure_package_config_file( - ${PROJECT_SOURCE_DIR}/hip-lang-config.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config.cmake - INSTALL_DESTINATION ${CONFIG_LANG_PACKAGE_INSTALL_DIR} - PATH_VARS LIB_INSTALL_DIR INCLUDE_INSTALL_DIR BIN_INSTALL_DIR) - -write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config-version.cmake - VERSION "${HIP_VERSION_MAJOR}.${HIP_VERSION_MINOR}.${HIP_VERSION_GITDATE}" - COMPATIBILITY SameMajorVersion) -install( - FILES - ${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config-version.cmake - DESTINATION - ${CONFIG_LANG_PACKAGE_INSTALL_DIR}/ - ) -endif() diff --git a/src/hipamd/cmake/FindROCclr.cmake b/src/hipamd/cmake/FindROCclr.cmake deleted file mode 100644 index 09b2ffa7cc..0000000000 --- a/src/hipamd/cmake/FindROCclr.cmake +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -if(ROCCLR_FOUND) - return() -endif() - -find_path(ROCCLR_INCLUDE_DIR top.hpp - HINTS - ${ROCCLR_PATH} - PATHS - # gerrit repo name - ${CMAKE_SOURCE_DIR}/vdi - ${CMAKE_SOURCE_DIR}/../vdi - ${CMAKE_SOURCE_DIR}/../../vdi - # github repo name - ${CMAKE_SOURCE_DIR}/ROCclr - ${CMAKE_SOURCE_DIR}/../ROCclr - ${CMAKE_SOURCE_DIR}/../../ROCclr - # jenkins repo name - ${CMAKE_SOURCE_DIR}/rocclr - ${CMAKE_SOURCE_DIR}/../rocclr - ${CMAKE_SOURCE_DIR}/../../rocclr - PATH_SUFFIXES - include) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ROCclr - "\nROCclr not found" - ROCCLR_INCLUDE_DIR) -mark_as_advanced(ROCCLR_INCLUDE_DIR) - -list(APPEND CMAKE_MODULE_PATH "${ROCCLR_INCLUDE_DIR}/../cmake") -include(ROCclr) diff --git a/src/hipamd/include/hip/amd_detail/amd_channel_descriptor.h b/src/hipamd/include/hip/amd_detail/amd_channel_descriptor.h deleted file mode 100644 index 7a282a61db..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_channel_descriptor.h +++ /dev/null @@ -1,348 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H - -#include -#include -#include - -#ifdef __cplusplus - -extern "C" HIP_PUBLIC_API -hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f); - -static inline hipChannelFormatDesc hipCreateChannelDescHalf() { - int e = (int)sizeof(unsigned short) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat); -} - -static inline hipChannelFormatDesc hipCreateChannelDescHalf1() { - int e = (int)sizeof(unsigned short) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat); -} - -static inline hipChannelFormatDesc hipCreateChannelDescHalf2() { - int e = (int)sizeof(unsigned short) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat); -} - -template -static inline hipChannelFormatDesc hipCreateChannelDesc() { - return hipCreateChannelDesc(0, 0, 0, 0, hipChannelFormatKindNone); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(char) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed char) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned char) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned char) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed char) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned char) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed char) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned); -} - -#ifndef __GNUC__ // vector3 is the same as vector4 -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned char) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed char) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned); -} -#endif - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned char) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed char) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned short) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed short) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned short) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed short) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned short) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed short) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned); -} - -#ifndef __GNUC__ -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned short) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed short) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned); -} -#endif - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned short) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed short) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned int) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed int) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned int) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed int) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned int) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed int) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned); -} - -#ifndef __GNUC__ -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned int) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed int) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned); -} -#endif - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned int) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed int) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(float) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(float) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(float) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindFloat); -} - -#ifndef __GNUC__ -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(float) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindFloat); -} -#endif - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(float) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindFloat); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned long) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed long) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned long) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed long) * 8; - return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned long) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed long) * 8; - return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned); -} - -#ifndef __GNUC__ -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned long) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed long) * 8; - return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned); -} -#endif - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(unsigned long) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned); -} - -template <> -inline hipChannelFormatDesc hipCreateChannelDesc() { - int e = (int)sizeof(signed long) * 8; - return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned); -} - -#else - -struct hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, - enum hipChannelFormatKind f); - -#endif - -#endif diff --git a/src/hipamd/include/hip/amd_detail/amd_device_functions.h b/src/hipamd/include/hip/amd_detail/amd_device_functions.h deleted file mode 100644 index 856a3bf829..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_device_functions.h +++ /dev/null @@ -1,1369 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_FUNCTIONS_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_FUNCTIONS_H - -#include "host_defines.h" -#include "math_fwd.h" - -#if !defined(__HIPCC_RTC__) -#include -#include -#endif // !defined(__HIPCC_RTC__) - -#include -#include -#include - -#if __HIP_CLANG_ONLY__ -extern "C" __device__ int printf(const char *fmt, ...); -#else -template -static inline __device__ void printf(const char* format, All... all) {} -#endif // __HIP_CLANG_ONLY__ - -/* -Integer Intrinsics -*/ - -// integer intrinsic function __poc __clz __ffs __brev -__device__ static inline unsigned int __popc(unsigned int input) { - return __builtin_popcount(input); -} -__device__ static inline unsigned int __popcll(unsigned long long int input) { - return __builtin_popcountll(input); -} - -__device__ static inline int __clz(int input) { - return __ockl_clz_u32((uint)input); -} - -__device__ static inline int __clzll(long long int input) { - return __ockl_clz_u64((uint64_t)input); -} - -__device__ static inline unsigned int __ffs(unsigned int input) { - return ( input == 0 ? -1 : __builtin_ctz(input) ) + 1; -} - -__device__ static inline unsigned int __ffsll(unsigned long long int input) { - return ( input == 0 ? -1 : __builtin_ctzll(input) ) + 1; -} - -__device__ static inline unsigned int __ffs(int input) { - return ( input == 0 ? -1 : __builtin_ctz(input) ) + 1; -} - -__device__ static inline unsigned int __ffsll(long long int input) { - return ( input == 0 ? -1 : __builtin_ctzll(input) ) + 1; -} - -__device__ static inline unsigned int __brev(unsigned int input) { - return __builtin_bitreverse32(input); -} - -__device__ static inline unsigned long long int __brevll(unsigned long long int input) { - return __builtin_bitreverse64(input); -} - -__device__ static inline unsigned int __lastbit_u32_u64(uint64_t input) { - return input == 0 ? -1 : __builtin_ctzl(input); -} - -__device__ static inline unsigned int __bitextract_u32(unsigned int src0, unsigned int src1, unsigned int src2) { - uint32_t offset = src1 & 31; - uint32_t width = src2 & 31; - return width == 0 ? 0 : (src0 << (32 - offset - width)) >> (32 - width); -} - -__device__ static inline uint64_t __bitextract_u64(uint64_t src0, unsigned int src1, unsigned int src2) { - uint64_t offset = src1 & 63; - uint64_t width = src2 & 63; - return width == 0 ? 0 : (src0 << (64 - offset - width)) >> (64 - width); -} - -__device__ static inline unsigned int __bitinsert_u32(unsigned int src0, unsigned int src1, unsigned int src2, unsigned int src3) { - uint32_t offset = src2 & 31; - uint32_t width = src3 & 31; - uint32_t mask = (1 << width) - 1; - return ((src0 & ~(mask << offset)) | ((src1 & mask) << offset)); -} - -__device__ static inline uint64_t __bitinsert_u64(uint64_t src0, uint64_t src1, unsigned int src2, unsigned int src3) { - uint64_t offset = src2 & 63; - uint64_t width = src3 & 63; - uint64_t mask = (1ULL << width) - 1; - return ((src0 & ~(mask << offset)) | ((src1 & mask) << offset)); -} - -__device__ inline unsigned int __funnelshift_l(unsigned int lo, unsigned int hi, unsigned int shift) -{ - uint32_t mask_shift = shift & 31; - return mask_shift == 0 ? hi : __builtin_amdgcn_alignbit(hi, lo, 32 - mask_shift); -} - -__device__ inline unsigned int __funnelshift_lc(unsigned int lo, unsigned int hi, unsigned int shift) -{ - uint32_t min_shift = shift >= 32 ? 32 : shift; - return min_shift == 0 ? hi : __builtin_amdgcn_alignbit(hi, lo, 32 - min_shift); -} - -__device__ inline unsigned int __funnelshift_r(unsigned int lo, unsigned int hi, unsigned int shift) -{ - return __builtin_amdgcn_alignbit(hi, lo, shift); -} - -__device__ inline unsigned int __funnelshift_rc(unsigned int lo, unsigned int hi, unsigned int shift) -{ - return shift >= 32 ? hi : __builtin_amdgcn_alignbit(hi, lo, shift); -} - -__device__ static unsigned int __byte_perm(unsigned int x, unsigned int y, unsigned int s); -__device__ static unsigned int __hadd(int x, int y); -__device__ static int __mul24(int x, int y); -__device__ static long long int __mul64hi(long long int x, long long int y); -__device__ static int __mulhi(int x, int y); -__device__ static int __rhadd(int x, int y); -__device__ static unsigned int __sad(int x, int y,unsigned int z); -__device__ static unsigned int __uhadd(unsigned int x, unsigned int y); -__device__ static int __umul24(unsigned int x, unsigned int y); -__device__ static unsigned long long int __umul64hi(unsigned long long int x, unsigned long long int y); -__device__ static unsigned int __umulhi(unsigned int x, unsigned int y); -__device__ static unsigned int __urhadd(unsigned int x, unsigned int y); -__device__ static unsigned int __usad(unsigned int x, unsigned int y, unsigned int z); - -struct ucharHolder { - union { - unsigned char c[4]; - unsigned int ui; - }; -} __attribute__((aligned(4))); - -struct uchar2Holder { - union { - unsigned int ui[2]; - unsigned char c[8]; - }; -} __attribute__((aligned(8))); - -__device__ -static inline unsigned int __byte_perm(unsigned int x, unsigned int y, unsigned int s) { - struct uchar2Holder cHoldVal; - struct ucharHolder cHoldKey; - cHoldKey.ui = s; - cHoldVal.ui[0] = x; - cHoldVal.ui[1] = y; - unsigned int result; - result = cHoldVal.c[cHoldKey.c[0] & 0x07]; - result += (cHoldVal.c[(cHoldKey.c[0] & 0x70) >> 4] << 8); - result += (cHoldVal.c[cHoldKey.c[1] & 0x07] << 16); - result += (cHoldVal.c[(cHoldKey.c[1] & 0x70) >> 4] << 24); - return result; -} - -__device__ static inline unsigned int __hadd(int x, int y) { - int z = x + y; - int sign = z & 0x8000000; - int value = z & 0x7FFFFFFF; - return ((value) >> 1 || sign); -} - -__device__ static inline int __mul24(int x, int y) { - return __ockl_mul24_i32(x, y); -} - -__device__ static inline long long __mul64hi(long long int x, long long int y) { - ulong x0 = (ulong)x & 0xffffffffUL; - long x1 = x >> 32; - ulong y0 = (ulong)y & 0xffffffffUL; - long y1 = y >> 32; - ulong z0 = x0*y0; - long t = x1*y0 + (z0 >> 32); - long z1 = t & 0xffffffffL; - long z2 = t >> 32; - z1 = x0*y1 + z1; - return x1*y1 + z2 + (z1 >> 32); -} - -__device__ static inline int __mulhi(int x, int y) { - return __ockl_mul_hi_i32(x, y); -} - -__device__ static inline int __rhadd(int x, int y) { - int z = x + y + 1; - int sign = z & 0x8000000; - int value = z & 0x7FFFFFFF; - return ((value) >> 1 || sign); -} -__device__ static inline unsigned int __sad(int x, int y, unsigned int z) { - return x > y ? x - y + z : y - x + z; -} -__device__ static inline unsigned int __uhadd(unsigned int x, unsigned int y) { - return (x + y) >> 1; -} -__device__ static inline int __umul24(unsigned int x, unsigned int y) { - return __ockl_mul24_u32(x, y); -} - -__device__ -static inline unsigned long long __umul64hi(unsigned long long int x, unsigned long long int y) { - ulong x0 = x & 0xffffffffUL; - ulong x1 = x >> 32; - ulong y0 = y & 0xffffffffUL; - ulong y1 = y >> 32; - ulong z0 = x0*y0; - ulong t = x1*y0 + (z0 >> 32); - ulong z1 = t & 0xffffffffUL; - ulong z2 = t >> 32; - z1 = x0*y1 + z1; - return x1*y1 + z2 + (z1 >> 32); -} - -__device__ static inline unsigned int __umulhi(unsigned int x, unsigned int y) { - return __ockl_mul_hi_u32(x, y); -} -__device__ static inline unsigned int __urhadd(unsigned int x, unsigned int y) { - return (x + y + 1) >> 1; -} -__device__ static inline unsigned int __usad(unsigned int x, unsigned int y, unsigned int z) { - return __ockl_sadd_u32(x, y, z); -} - -__device__ static inline unsigned int __lane_id() { - return __builtin_amdgcn_mbcnt_hi( - -1, __builtin_amdgcn_mbcnt_lo(-1, 0)); -} - -__device__ -static inline unsigned int __mbcnt_lo(unsigned int x, unsigned int y) {return __builtin_amdgcn_mbcnt_lo(x,y);}; - -__device__ -static inline unsigned int __mbcnt_hi(unsigned int x, unsigned int y) {return __builtin_amdgcn_mbcnt_hi(x,y);}; - -/* -HIP specific device functions -*/ - -__device__ static inline unsigned __hip_ds_bpermute(int index, unsigned src) { - union { int i; unsigned u; float f; } tmp; tmp.u = src; - tmp.i = __builtin_amdgcn_ds_bpermute(index, tmp.i); - return tmp.u; -} - -__device__ static inline float __hip_ds_bpermutef(int index, float src) { - union { int i; unsigned u; float f; } tmp; tmp.f = src; - tmp.i = __builtin_amdgcn_ds_bpermute(index, tmp.i); - return tmp.f; -} - -__device__ static inline unsigned __hip_ds_permute(int index, unsigned src) { - union { int i; unsigned u; float f; } tmp; tmp.u = src; - tmp.i = __builtin_amdgcn_ds_permute(index, tmp.i); - return tmp.u; -} - -__device__ static inline float __hip_ds_permutef(int index, float src) { - union { int i; unsigned u; float f; } tmp; tmp.u = src; - tmp.i = __builtin_amdgcn_ds_permute(index, tmp.i); - return tmp.u; -} - -#define __hip_ds_swizzle(src, pattern) __hip_ds_swizzle_N<(pattern)>((src)) -#define __hip_ds_swizzlef(src, pattern) __hip_ds_swizzlef_N<(pattern)>((src)) - -template -__device__ static inline unsigned __hip_ds_swizzle_N(unsigned int src) { - union { int i; unsigned u; float f; } tmp; tmp.u = src; - tmp.i = __builtin_amdgcn_ds_swizzle(tmp.i, pattern); - return tmp.u; -} - -template -__device__ static inline float __hip_ds_swizzlef_N(float src) { - union { int i; unsigned u; float f; } tmp; tmp.f = src; - tmp.i = __builtin_amdgcn_ds_swizzle(tmp.i, pattern); - return tmp.f; -} - -#define __hip_move_dpp(src, dpp_ctrl, row_mask, bank_mask, bound_ctrl) \ - __hip_move_dpp_N<(dpp_ctrl), (row_mask), (bank_mask), (bound_ctrl)>((src)) - -template -__device__ static inline int __hip_move_dpp_N(int src) { - return __builtin_amdgcn_mov_dpp(src, dpp_ctrl, row_mask, bank_mask, - bound_ctrl); -} - -static constexpr int warpSize = __AMDGCN_WAVEFRONT_SIZE; - -__device__ -inline -int __shfl(int var, int src_lane, int width = warpSize) { - int self = __lane_id(); - int index = src_lane + (self & ~(width-1)); - return __builtin_amdgcn_ds_bpermute(index<<2, var); -} -__device__ -inline -unsigned int __shfl(unsigned int var, int src_lane, int width = warpSize) { - union { int i; unsigned u; float f; } tmp; tmp.u = var; - tmp.i = __shfl(tmp.i, src_lane, width); - return tmp.u; -} -__device__ -inline -float __shfl(float var, int src_lane, int width = warpSize) { - union { int i; unsigned u; float f; } tmp; tmp.f = var; - tmp.i = __shfl(tmp.i, src_lane, width); - return tmp.f; -} -__device__ -inline -double __shfl(double var, int src_lane, int width = warpSize) { - static_assert(sizeof(double) == 2 * sizeof(int), ""); - static_assert(sizeof(double) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl(tmp[0], src_lane, width); - tmp[1] = __shfl(tmp[1], src_lane, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - double tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} -__device__ -inline -long __shfl(long var, int src_lane, int width = warpSize) -{ - #ifndef _MSC_VER - static_assert(sizeof(long) == 2 * sizeof(int), ""); - static_assert(sizeof(long) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl(tmp[0], src_lane, width); - tmp[1] = __shfl(tmp[1], src_lane, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; - #else - static_assert(sizeof(long) == sizeof(int), ""); - return static_cast(__shfl(static_cast(var), src_lane, width)); - #endif -} -__device__ -inline -unsigned long __shfl(unsigned long var, int src_lane, int width = warpSize) { - #ifndef _MSC_VER - static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), ""); - static_assert(sizeof(unsigned long) == sizeof(uint64_t), ""); - - unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl(tmp[0], src_lane, width); - tmp[1] = __shfl(tmp[1], src_lane, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - unsigned long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; - #else - static_assert(sizeof(unsigned long) == sizeof(unsigned int), ""); - return static_cast(__shfl(static_cast(var), src_lane, width)); - #endif -} -__device__ -inline -long long __shfl(long long var, int src_lane, int width = warpSize) -{ - static_assert(sizeof(long long) == 2 * sizeof(int), ""); - static_assert(sizeof(long long) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl(tmp[0], src_lane, width); - tmp[1] = __shfl(tmp[1], src_lane, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} -__device__ -inline -unsigned long long __shfl(unsigned long long var, int src_lane, int width = warpSize) { - static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), ""); - static_assert(sizeof(unsigned long long) == sizeof(uint64_t), ""); - - unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl(tmp[0], src_lane, width); - tmp[1] = __shfl(tmp[1], src_lane, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - unsigned long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} - -__device__ -inline -int __shfl_up(int var, unsigned int lane_delta, int width = warpSize) { - int self = __lane_id(); - int index = self - lane_delta; - index = (index < (self & ~(width-1)))?self:index; - return __builtin_amdgcn_ds_bpermute(index<<2, var); -} -__device__ -inline -unsigned int __shfl_up(unsigned int var, unsigned int lane_delta, int width = warpSize) { - union { int i; unsigned u; float f; } tmp; tmp.u = var; - tmp.i = __shfl_up(tmp.i, lane_delta, width); - return tmp.u; -} -__device__ -inline -float __shfl_up(float var, unsigned int lane_delta, int width = warpSize) { - union { int i; unsigned u; float f; } tmp; tmp.f = var; - tmp.i = __shfl_up(tmp.i, lane_delta, width); - return tmp.f; -} -__device__ -inline -double __shfl_up(double var, unsigned int lane_delta, int width = warpSize) { - static_assert(sizeof(double) == 2 * sizeof(int), ""); - static_assert(sizeof(double) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_up(tmp[0], lane_delta, width); - tmp[1] = __shfl_up(tmp[1], lane_delta, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - double tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} -__device__ -inline -long __shfl_up(long var, unsigned int lane_delta, int width = warpSize) -{ - #ifndef _MSC_VER - static_assert(sizeof(long) == 2 * sizeof(int), ""); - static_assert(sizeof(long) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_up(tmp[0], lane_delta, width); - tmp[1] = __shfl_up(tmp[1], lane_delta, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; - #else - static_assert(sizeof(long) == sizeof(int), ""); - return static_cast(__shfl_up(static_cast(var), lane_delta, width)); - #endif -} - -__device__ -inline -unsigned long __shfl_up(unsigned long var, unsigned int lane_delta, int width = warpSize) -{ - #ifndef _MSC_VER - static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), ""); - static_assert(sizeof(unsigned long) == sizeof(uint64_t), ""); - - unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_up(tmp[0], lane_delta, width); - tmp[1] = __shfl_up(tmp[1], lane_delta, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - unsigned long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; - #else - static_assert(sizeof(unsigned long) == sizeof(unsigned int), ""); - return static_cast(__shfl_up(static_cast(var), lane_delta, width)); - #endif -} - -__device__ -inline -long long __shfl_up(long long var, unsigned int lane_delta, int width = warpSize) -{ - static_assert(sizeof(long long) == 2 * sizeof(int), ""); - static_assert(sizeof(long long) == sizeof(uint64_t), ""); - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_up(tmp[0], lane_delta, width); - tmp[1] = __shfl_up(tmp[1], lane_delta, width); - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} - -__device__ -inline -unsigned long long __shfl_up(unsigned long long var, unsigned int lane_delta, int width = warpSize) -{ - static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), ""); - static_assert(sizeof(unsigned long long) == sizeof(uint64_t), ""); - unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_up(tmp[0], lane_delta, width); - tmp[1] = __shfl_up(tmp[1], lane_delta, width); - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - unsigned long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} - -__device__ -inline -int __shfl_down(int var, unsigned int lane_delta, int width = warpSize) { - int self = __lane_id(); - int index = self + lane_delta; - index = (int)((self&(width-1))+lane_delta) >= width?self:index; - return __builtin_amdgcn_ds_bpermute(index<<2, var); -} -__device__ -inline -unsigned int __shfl_down(unsigned int var, unsigned int lane_delta, int width = warpSize) { - union { int i; unsigned u; float f; } tmp; tmp.u = var; - tmp.i = __shfl_down(tmp.i, lane_delta, width); - return tmp.u; -} -__device__ -inline -float __shfl_down(float var, unsigned int lane_delta, int width = warpSize) { - union { int i; unsigned u; float f; } tmp; tmp.f = var; - tmp.i = __shfl_down(tmp.i, lane_delta, width); - return tmp.f; -} -__device__ -inline -double __shfl_down(double var, unsigned int lane_delta, int width = warpSize) { - static_assert(sizeof(double) == 2 * sizeof(int), ""); - static_assert(sizeof(double) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_down(tmp[0], lane_delta, width); - tmp[1] = __shfl_down(tmp[1], lane_delta, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - double tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} -__device__ -inline -long __shfl_down(long var, unsigned int lane_delta, int width = warpSize) -{ - #ifndef _MSC_VER - static_assert(sizeof(long) == 2 * sizeof(int), ""); - static_assert(sizeof(long) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_down(tmp[0], lane_delta, width); - tmp[1] = __shfl_down(tmp[1], lane_delta, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; - #else - static_assert(sizeof(long) == sizeof(int), ""); - return static_cast(__shfl_down(static_cast(var), lane_delta, width)); - #endif -} -__device__ -inline -unsigned long __shfl_down(unsigned long var, unsigned int lane_delta, int width = warpSize) -{ - #ifndef _MSC_VER - static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), ""); - static_assert(sizeof(unsigned long) == sizeof(uint64_t), ""); - - unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_down(tmp[0], lane_delta, width); - tmp[1] = __shfl_down(tmp[1], lane_delta, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - unsigned long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; - #else - static_assert(sizeof(unsigned long) == sizeof(unsigned int), ""); - return static_cast(__shfl_down(static_cast(var), lane_delta, width)); - #endif -} -__device__ -inline -long long __shfl_down(long long var, unsigned int lane_delta, int width = warpSize) -{ - static_assert(sizeof(long long) == 2 * sizeof(int), ""); - static_assert(sizeof(long long) == sizeof(uint64_t), ""); - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_down(tmp[0], lane_delta, width); - tmp[1] = __shfl_down(tmp[1], lane_delta, width); - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} -__device__ -inline -unsigned long long __shfl_down(unsigned long long var, unsigned int lane_delta, int width = warpSize) -{ - static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), ""); - static_assert(sizeof(unsigned long long) == sizeof(uint64_t), ""); - unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_down(tmp[0], lane_delta, width); - tmp[1] = __shfl_down(tmp[1], lane_delta, width); - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - unsigned long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} - -__device__ -inline -int __shfl_xor(int var, int lane_mask, int width = warpSize) { - int self = __lane_id(); - int index = self^lane_mask; - index = index >= ((self+width)&~(width-1))?self:index; - return __builtin_amdgcn_ds_bpermute(index<<2, var); -} -__device__ -inline -unsigned int __shfl_xor(unsigned int var, int lane_mask, int width = warpSize) { - union { int i; unsigned u; float f; } tmp; tmp.u = var; - tmp.i = __shfl_xor(tmp.i, lane_mask, width); - return tmp.u; -} -__device__ -inline -float __shfl_xor(float var, int lane_mask, int width = warpSize) { - union { int i; unsigned u; float f; } tmp; tmp.f = var; - tmp.i = __shfl_xor(tmp.i, lane_mask, width); - return tmp.f; -} -__device__ -inline -double __shfl_xor(double var, int lane_mask, int width = warpSize) { - static_assert(sizeof(double) == 2 * sizeof(int), ""); - static_assert(sizeof(double) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_xor(tmp[0], lane_mask, width); - tmp[1] = __shfl_xor(tmp[1], lane_mask, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - double tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} -__device__ -inline -long __shfl_xor(long var, int lane_mask, int width = warpSize) -{ - #ifndef _MSC_VER - static_assert(sizeof(long) == 2 * sizeof(int), ""); - static_assert(sizeof(long) == sizeof(uint64_t), ""); - - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_xor(tmp[0], lane_mask, width); - tmp[1] = __shfl_xor(tmp[1], lane_mask, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; - #else - static_assert(sizeof(long) == sizeof(int), ""); - return static_cast(__shfl_xor(static_cast(var), lane_mask, width)); - #endif -} -__device__ -inline -unsigned long __shfl_xor(unsigned long var, int lane_mask, int width = warpSize) -{ - #ifndef _MSC_VER - static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), ""); - static_assert(sizeof(unsigned long) == sizeof(uint64_t), ""); - - unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_xor(tmp[0], lane_mask, width); - tmp[1] = __shfl_xor(tmp[1], lane_mask, width); - - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - unsigned long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; - #else - static_assert(sizeof(unsigned long) == sizeof(unsigned int), ""); - return static_cast(__shfl_xor(static_cast(var), lane_mask, width)); - #endif -} -__device__ -inline -long long __shfl_xor(long long var, int lane_mask, int width = warpSize) -{ - static_assert(sizeof(long long) == 2 * sizeof(int), ""); - static_assert(sizeof(long long) == sizeof(uint64_t), ""); - int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_xor(tmp[0], lane_mask, width); - tmp[1] = __shfl_xor(tmp[1], lane_mask, width); - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} -__device__ -inline -unsigned long long __shfl_xor(unsigned long long var, int lane_mask, int width = warpSize) -{ - static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), ""); - static_assert(sizeof(unsigned long long) == sizeof(uint64_t), ""); - unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); - tmp[0] = __shfl_xor(tmp[0], lane_mask, width); - tmp[1] = __shfl_xor(tmp[1], lane_mask, width); - uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); - unsigned long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - return tmp1; -} -#define MASK1 0x00ff00ff -#define MASK2 0xff00ff00 - -__device__ static inline char4 __hip_hc_add8pk(char4 in1, char4 in2) { - char4 out; - unsigned one1 = in1.w & MASK1; - unsigned one2 = in2.w & MASK1; - out.w = (one1 + one2) & MASK1; - one1 = in1.w & MASK2; - one2 = in2.w & MASK2; - out.w = out.w | ((one1 + one2) & MASK2); - return out; -} - -__device__ static inline char4 __hip_hc_sub8pk(char4 in1, char4 in2) { - char4 out; - unsigned one1 = in1.w & MASK1; - unsigned one2 = in2.w & MASK1; - out.w = (one1 - one2) & MASK1; - one1 = in1.w & MASK2; - one2 = in2.w & MASK2; - out.w = out.w | ((one1 - one2) & MASK2); - return out; -} - -__device__ static inline char4 __hip_hc_mul8pk(char4 in1, char4 in2) { - char4 out; - unsigned one1 = in1.w & MASK1; - unsigned one2 = in2.w & MASK1; - out.w = (one1 * one2) & MASK1; - one1 = in1.w & MASK2; - one2 = in2.w & MASK2; - out.w = out.w | ((one1 * one2) & MASK2); - return out; -} - -/* - * Rounding modes are not yet supported in HIP - * TODO: Conversion functions are not correct, need to fix when BE is ready -*/ - -__device__ static inline float __double2float_rd(double x) { return (double)x; } -__device__ static inline float __double2float_rn(double x) { return (double)x; } -__device__ static inline float __double2float_ru(double x) { return (double)x; } -__device__ static inline float __double2float_rz(double x) { return (double)x; } - -__device__ static inline int __double2hiint(double x) { - static_assert(sizeof(double) == 2 * sizeof(int), ""); - - int tmp[2]; - __builtin_memcpy(tmp, &x, sizeof(tmp)); - - return tmp[1]; -} -__device__ static inline int __double2loint(double x) { - static_assert(sizeof(double) == 2 * sizeof(int), ""); - - int tmp[2]; - __builtin_memcpy(tmp, &x, sizeof(tmp)); - - return tmp[0]; -} - -__device__ static inline int __double2int_rd(double x) { return (int)__ocml_floor_f64(x); } -__device__ static inline int __double2int_rn(double x) { return (int)__ocml_rint_f64(x); } -__device__ static inline int __double2int_ru(double x) { return (int)__ocml_ceil_f64(x); } -__device__ static inline int __double2int_rz(double x) { return (int)x; } - -__device__ static inline long long int __double2ll_rd(double x) { - return (long long)__ocml_floor_f64(x); -} -__device__ static inline long long int __double2ll_rn(double x) { - return (long long)__ocml_rint_f64(x); -} -__device__ static inline long long int __double2ll_ru(double x) { - return (long long)__ocml_ceil_f64(x); -} -__device__ static inline long long int __double2ll_rz(double x) { return (long long)x; } - -__device__ static inline unsigned int __double2uint_rd(double x) { - return (unsigned int)__ocml_floor_f64(x); -} -__device__ static inline unsigned int __double2uint_rn(double x) { - return (unsigned int)__ocml_rint_f64(x); -} -__device__ static inline unsigned int __double2uint_ru(double x) { - return (unsigned int)__ocml_ceil_f64(x); -} -__device__ static inline unsigned int __double2uint_rz(double x) { return (unsigned int)x; } - -__device__ static inline unsigned long long int __double2ull_rd(double x) { - return (unsigned long long int)__ocml_floor_f64(x); -} -__device__ static inline unsigned long long int __double2ull_rn(double x) { - return (unsigned long long int)__ocml_rint_f64(x); -} -__device__ static inline unsigned long long int __double2ull_ru(double x) { - return (unsigned long long int)__ocml_ceil_f64(x); -} -__device__ static inline unsigned long long int __double2ull_rz(double x) { - return (unsigned long long int)x; -} - -__device__ static inline long long int __double_as_longlong(double x) { - static_assert(sizeof(long long) == sizeof(double), ""); - - long long tmp; - __builtin_memcpy(&tmp, &x, sizeof(tmp)); - - return tmp; -} - -/* -__device__ unsigned short __float2half_rn(float x); -__device__ float __half2float(unsigned short); - -The above device function are not a valid . -Use -__device__ __half __float2half_rn(float x); -__device__ float __half2float(__half); -from hip_fp16.h - -CUDA implements half as unsigned short whereas, HIP doesn't. - -*/ - -__device__ static inline int __float2int_rd(float x) { return (int)__ocml_floor_f32(x); } -__device__ static inline int __float2int_rn(float x) { return (int)__ocml_rint_f32(x); } -__device__ static inline int __float2int_ru(float x) { return (int)__ocml_ceil_f32(x); } -__device__ static inline int __float2int_rz(float x) { return (int)__ocml_trunc_f32(x); } - -__device__ static inline long long int __float2ll_rd(float x) { - return (long long int)__ocml_floor_f32(x); -} -__device__ static inline long long int __float2ll_rn(float x) { - return (long long int)__ocml_rint_f32(x); -} -__device__ static inline long long int __float2ll_ru(float x) { - return (long long int)__ocml_ceil_f32(x); -} -__device__ static inline long long int __float2ll_rz(float x) { return (long long int)x; } - -__device__ static inline unsigned int __float2uint_rd(float x) { - return (unsigned int)__ocml_floor_f32(x); -} -__device__ static inline unsigned int __float2uint_rn(float x) { - return (unsigned int)__ocml_rint_f32(x); -} -__device__ static inline unsigned int __float2uint_ru(float x) { - return (unsigned int)__ocml_ceil_f32(x); -} -__device__ static inline unsigned int __float2uint_rz(float x) { return (unsigned int)x; } - -__device__ static inline unsigned long long int __float2ull_rd(float x) { - return (unsigned long long int)__ocml_floor_f32(x); -} -__device__ static inline unsigned long long int __float2ull_rn(float x) { - return (unsigned long long int)__ocml_rint_f32(x); -} -__device__ static inline unsigned long long int __float2ull_ru(float x) { - return (unsigned long long int)__ocml_ceil_f32(x); -} -__device__ static inline unsigned long long int __float2ull_rz(float x) { - return (unsigned long long int)x; -} - -__device__ static inline int __float_as_int(float x) { - static_assert(sizeof(int) == sizeof(float), ""); - - int tmp; - __builtin_memcpy(&tmp, &x, sizeof(tmp)); - - return tmp; -} - -__device__ static inline unsigned int __float_as_uint(float x) { - static_assert(sizeof(unsigned int) == sizeof(float), ""); - - unsigned int tmp; - __builtin_memcpy(&tmp, &x, sizeof(tmp)); - - return tmp; -} - -__device__ static inline double __hiloint2double(int hi, int lo) { - static_assert(sizeof(double) == sizeof(uint64_t), ""); - - uint64_t tmp0 = (static_cast(hi) << 32ull) | static_cast(lo); - double tmp1; - __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); - - return tmp1; -} - -__device__ static inline double __int2double_rn(int x) { return (double)x; } - -__device__ static inline float __int2float_rd(int x) { return (float)x; } -__device__ static inline float __int2float_rn(int x) { return (float)x; } -__device__ static inline float __int2float_ru(int x) { return (float)x; } -__device__ static inline float __int2float_rz(int x) { return (float)x; } - -__device__ static inline float __int_as_float(int x) { - static_assert(sizeof(float) == sizeof(int), ""); - - float tmp; - __builtin_memcpy(&tmp, &x, sizeof(tmp)); - - return tmp; -} - -__device__ static inline double __ll2double_rd(long long int x) { return (double)x; } -__device__ static inline double __ll2double_rn(long long int x) { return (double)x; } -__device__ static inline double __ll2double_ru(long long int x) { return (double)x; } -__device__ static inline double __ll2double_rz(long long int x) { return (double)x; } - -__device__ static inline float __ll2float_rd(long long int x) { return (float)x; } -__device__ static inline float __ll2float_rn(long long int x) { return (float)x; } -__device__ static inline float __ll2float_ru(long long int x) { return (float)x; } -__device__ static inline float __ll2float_rz(long long int x) { return (float)x; } - -__device__ static inline double __longlong_as_double(long long int x) { - static_assert(sizeof(double) == sizeof(long long), ""); - - double tmp; - __builtin_memcpy(&tmp, &x, sizeof(tmp)); - - return tmp; -} - -__device__ static inline double __uint2double_rn(int x) { return (double)x; } - -__device__ static inline float __uint2float_rd(unsigned int x) { return (float)x; } -__device__ static inline float __uint2float_rn(unsigned int x) { return (float)x; } -__device__ static inline float __uint2float_ru(unsigned int x) { return (float)x; } -__device__ static inline float __uint2float_rz(unsigned int x) { return (float)x; } - -__device__ static inline float __uint_as_float(unsigned int x) { - static_assert(sizeof(float) == sizeof(unsigned int), ""); - - float tmp; - __builtin_memcpy(&tmp, &x, sizeof(tmp)); - - return tmp; -} - -__device__ static inline double __ull2double_rd(unsigned long long int x) { return (double)x; } -__device__ static inline double __ull2double_rn(unsigned long long int x) { return (double)x; } -__device__ static inline double __ull2double_ru(unsigned long long int x) { return (double)x; } -__device__ static inline double __ull2double_rz(unsigned long long int x) { return (double)x; } - -__device__ static inline float __ull2float_rd(unsigned long long int x) { return (float)x; } -__device__ static inline float __ull2float_rn(unsigned long long int x) { return (float)x; } -__device__ static inline float __ull2float_ru(unsigned long long int x) { return (float)x; } -__device__ static inline float __ull2float_rz(unsigned long long int x) { return (float)x; } - -#if __HIP_CLANG_ONLY__ - -// Clock functions -__device__ long long int __clock64(); -__device__ long long int __clock(); -__device__ long long int clock64(); -__device__ long long int clock(); -// hip.amdgcn.bc - named sync -__device__ void __named_sync(int a, int b); - -#ifdef __HIP_DEVICE_COMPILE__ - -// Clock function to return GPU core cycle count. -// GPU can change its core clock frequency at runtime. The maximum frequency can be queried -// through hipDeviceAttributeClockRate attribute. -__device__ -inline __attribute((always_inline)) -long long int __clock64() { -#if __has_builtin(__builtin_amdgcn_s_memtime) - // Exists on gfx8, gfx9, gfx10.1, gfx10.2, gfx10.3 - return (long long int) __builtin_amdgcn_s_memtime(); -#else - // Subject to change when better solution available - return (long long int) __builtin_readcyclecounter(); -#endif -} - -__device__ -inline __attribute((always_inline)) -long long int __clock() { return __clock64(); } - -// Clock function to return wall clock count at a constant frequency. The interface to query -// the frequency will be implemented. -__device__ -inline __attribute__((always_inline)) -long long int wall_clock64() { -#if __has_builtin(__builtin_amdgcn_s_memrealtime) - // Exists since gfx8 - return (long long int) __builtin_amdgcn_s_memrealtime(); -#else - return -1; // Negative return means __builtin_amdgcn_s_memrealtime unavailable. -#endif -} - -__device__ -inline __attribute__((always_inline)) -long long int clock64() { return __clock64(); } - -__device__ -inline __attribute__((always_inline)) -long long int clock() { return __clock(); } - -// hip.amdgcn.bc - named sync -__device__ -inline -void __named_sync(int a, int b) { __builtin_amdgcn_s_barrier(); } - -#endif // __HIP_DEVICE_COMPILE__ - -// warp vote function __all __any __ballot -__device__ -inline -int __all(int predicate) { - return __ockl_wfall_i32(predicate); -} - -__device__ -inline -int __any(int predicate) { - return __ockl_wfany_i32(predicate); -} - -// XXX from llvm/include/llvm/IR/InstrTypes.h -#define ICMP_NE 33 - -__device__ -inline -unsigned long long int __ballot(int predicate) { - return __builtin_amdgcn_uicmp(predicate, 0, ICMP_NE); -} - -__device__ -inline -unsigned long long int __ballot64(int predicate) { - return __builtin_amdgcn_uicmp(predicate, 0, ICMP_NE); -} - -// hip.amdgcn.bc - lanemask -__device__ -inline -uint64_t __lanemask_gt() -{ - uint32_t lane = __ockl_lane_u32(); - if (lane == 63) - return 0; - uint64_t ballot = __ballot64(1); - uint64_t mask = (~((uint64_t)0)) << (lane + 1); - return mask & ballot; -} - -__device__ -inline -uint64_t __lanemask_lt() -{ - uint32_t lane = __ockl_lane_u32(); - int64_t ballot = __ballot64(1); - uint64_t mask = ((uint64_t)1 << lane) - (uint64_t)1; - return mask & ballot; -} - -__device__ -inline -uint64_t __lanemask_eq() -{ - uint32_t lane = __ockl_lane_u32(); - int64_t mask = ((uint64_t)1 << lane); - return mask; -} - - -__device__ inline void* __local_to_generic(void* p) { return p; } - -#ifdef __HIP_DEVICE_COMPILE__ -__device__ -inline -void* __get_dynamicgroupbaseptr() -{ - // Get group segment base pointer. - return (char*)__local_to_generic((void*)__to_local(__llvm_amdgcn_groupstaticsize())); -} -#else -__device__ -void* __get_dynamicgroupbaseptr(); -#endif // __HIP_DEVICE_COMPILE__ - -__device__ -inline -void *__amdgcn_get_dynamicgroupbaseptr() { - return __get_dynamicgroupbaseptr(); -} - -// Memory Fence Functions -__device__ -inline -static void __threadfence() -{ - __atomic_work_item_fence(0, __memory_order_seq_cst, __memory_scope_device); -} - -__device__ -inline -static void __threadfence_block() -{ - __atomic_work_item_fence(0, __memory_order_seq_cst, __memory_scope_work_group); -} - -__device__ -inline -static void __threadfence_system() -{ - __atomic_work_item_fence(0, __memory_order_seq_cst, __memory_scope_all_svm_devices); -} - -// abort -__device__ -inline -__attribute__((weak)) -void abort() { - return __builtin_trap(); -} - -// The noinline attribute helps encapsulate the printf expansion, -// which otherwise has a performance impact just by increasing the -// size of the calling function. Additionally, the weak attribute -// allows the function to exist as a global although its definition is -// included in every compilation unit. -#if defined(_WIN32) || defined(_WIN64) -extern "C" __device__ __attribute__((noinline)) __attribute__((weak)) -void _wassert(const wchar_t *_msg, const wchar_t *_file, unsigned _line) { - // FIXME: Need `wchar_t` support to generate assertion message. - __builtin_trap(); -} -#else /* defined(_WIN32) || defined(_WIN64) */ -extern "C" __device__ __attribute__((noinline)) __attribute__((weak)) -void __assert_fail(const char *assertion, - const char *file, - unsigned int line, - const char *function) -{ - printf("%s:%u: %s: Device-side assertion `%s' failed.\n", file, line, - function, assertion); - __builtin_trap(); -} - -extern "C" __device__ __attribute__((noinline)) __attribute__((weak)) -void __assertfail(const char *assertion, - const char *file, - unsigned int line, - const char *function, - size_t charsize) -{ - // ignore all the args for now. - __builtin_trap(); -} -#endif /* defined(_WIN32) || defined(_WIN64) */ - -__device__ -inline -static void __work_group_barrier(__cl_mem_fence_flags flags, __memory_scope scope) -{ - if (flags) { - __atomic_work_item_fence(flags, __memory_order_release, scope); - __builtin_amdgcn_s_barrier(); - __atomic_work_item_fence(flags, __memory_order_acquire, scope); - } else { - __builtin_amdgcn_s_barrier(); - } -} - -__device__ -inline -static void __barrier(int n) -{ - __work_group_barrier((__cl_mem_fence_flags)n, __memory_scope_work_group); -} - -__device__ -inline -__attribute__((convergent)) -void __syncthreads() -{ - __barrier(__CLK_LOCAL_MEM_FENCE); -} - -__device__ -inline -__attribute__((convergent)) -int __syncthreads_count(int predicate) -{ - return __ockl_wgred_add_i32(!!predicate); -} - -__device__ -inline -__attribute__((convergent)) -int __syncthreads_and(int predicate) -{ - return __ockl_wgred_and_i32(!!predicate); -} - -__device__ -inline -__attribute__((convergent)) -int __syncthreads_or(int predicate) -{ - return __ockl_wgred_or_i32(!!predicate); -} - -// hip.amdgcn.bc - device routine -/* - HW_ID Register bit structure - WAVE_ID 3:0 Wave buffer slot number. 0-9. - SIMD_ID 5:4 SIMD which the wave is assigned to within the CU. - PIPE_ID 7:6 Pipeline from which the wave was dispatched. - CU_ID 11:8 Compute Unit the wave is assigned to. - SH_ID 12 Shader Array (within an SE) the wave is assigned to. - SE_ID 14:13 Shader Engine the wave is assigned to. - TG_ID 19:16 Thread-group ID - VM_ID 23:20 Virtual Memory ID - QUEUE_ID 26:24 Queue from which this wave was dispatched. - STATE_ID 29:27 State ID (graphics only, not compute). - ME_ID 31:30 Micro-engine ID. - */ - -#define HW_ID 4 - -#define HW_ID_CU_ID_SIZE 4 -#define HW_ID_CU_ID_OFFSET 8 - -#define HW_ID_SE_ID_SIZE 2 -#define HW_ID_SE_ID_OFFSET 13 - -/* - Encoding of parameter bitmask - HW_ID 5:0 HW_ID - OFFSET 10:6 Range: 0..31 - SIZE 15:11 Range: 1..32 - */ - -#define GETREG_IMMED(SZ,OFF,REG) (((SZ) << 11) | ((OFF) << 6) | (REG)) - -/* - __smid returns the wave's assigned Compute Unit and Shader Engine. - The Compute Unit, CU_ID returned in bits 3:0, and Shader Engine, SE_ID in bits 5:4. - Note: the results vary over time. - SZ minus 1 since SIZE is 1-based. -*/ -__device__ -inline -unsigned __smid(void) -{ - unsigned cu_id = __builtin_amdgcn_s_getreg( - GETREG_IMMED(HW_ID_CU_ID_SIZE-1, HW_ID_CU_ID_OFFSET, HW_ID)); - unsigned se_id = __builtin_amdgcn_s_getreg( - GETREG_IMMED(HW_ID_SE_ID_SIZE-1, HW_ID_SE_ID_OFFSET, HW_ID)); - - /* Each shader engine has 16 CU */ - return (se_id << HW_ID_CU_ID_SIZE) + cu_id; -} - -/** - * Map HIP_DYNAMIC_SHARED to "extern __shared__" for compatibility with old HIP applications - * To be removed in a future release. - */ -#define HIP_DYNAMIC_SHARED(type, var) extern __shared__ type var[]; -#define HIP_DYNAMIC_SHARED_ATTRIBUTE - -#endif //defined(__clang__) && defined(__HIP__) - - -// loop unrolling -static inline __device__ void* __hip_hc_memcpy(void* dst, const void* src, size_t size) { - auto dstPtr = static_cast(dst); - auto srcPtr = static_cast(src); - - while (size >= 4u) { - dstPtr[0] = srcPtr[0]; - dstPtr[1] = srcPtr[1]; - dstPtr[2] = srcPtr[2]; - dstPtr[3] = srcPtr[3]; - - size -= 4u; - srcPtr += 4u; - dstPtr += 4u; - } - switch (size) { - case 3: - dstPtr[2] = srcPtr[2]; - case 2: - dstPtr[1] = srcPtr[1]; - case 1: - dstPtr[0] = srcPtr[0]; - } - - return dst; -} - -static inline __device__ void* __hip_hc_memset(void* dst, unsigned char val, size_t size) { - auto dstPtr = static_cast(dst); - - while (size >= 4u) { - dstPtr[0] = val; - dstPtr[1] = val; - dstPtr[2] = val; - dstPtr[3] = val; - - size -= 4u; - dstPtr += 4u; - } - switch (size) { - case 3: - dstPtr[2] = val; - case 2: - dstPtr[1] = val; - case 1: - dstPtr[0] = val; - } - - return dst; -} -#ifndef __OPENMP_AMDGCN__ -static inline __device__ void* memcpy(void* dst, const void* src, size_t size) { - return __hip_hc_memcpy(dst, src, size); -} - -static inline __device__ void* memset(void* ptr, int val, size_t size) { - unsigned char val8 = static_cast(val); - return __hip_hc_memset(ptr, val8, size); -} -#endif // !__OPENMP_AMDGCN__ -#endif diff --git a/src/hipamd/include/hip/amd_detail/amd_driver_types.h b/src/hipamd/include/hip/amd_detail/amd_driver_types.h deleted file mode 100644 index fe29d1f144..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_driver_types.h +++ /dev/null @@ -1,478 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_DRIVER_TYPES_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_DRIVER_TYPES_H - -// The follow macro should be removed after upstream updation. -// It's defined here for workarround of rocThrust building failure. -#define HIP_INCLUDE_HIP_HCC_DETAIL_DRIVER_TYPES_H - -#if !defined(__HIPCC_RTC__) -#ifndef __cplusplus -#include -#endif -#endif // !defined(__HIPCC_RTC__) - -typedef void* hipDeviceptr_t; -typedef enum hipChannelFormatKind { - hipChannelFormatKindSigned = 0, - hipChannelFormatKindUnsigned = 1, - hipChannelFormatKindFloat = 2, - hipChannelFormatKindNone = 3 -}hipChannelFormatKind; - -typedef struct hipChannelFormatDesc { - int x; - int y; - int z; - int w; - enum hipChannelFormatKind f; -}hipChannelFormatDesc; - -#define HIP_TRSA_OVERRIDE_FORMAT 0x01 -#define HIP_TRSF_READ_AS_INTEGER 0x01 -#define HIP_TRSF_NORMALIZED_COORDINATES 0x02 -#define HIP_TRSF_SRGB 0x10 - -typedef enum hipArray_Format { - HIP_AD_FORMAT_UNSIGNED_INT8 = 0x01, - HIP_AD_FORMAT_UNSIGNED_INT16 = 0x02, - HIP_AD_FORMAT_UNSIGNED_INT32 = 0x03, - HIP_AD_FORMAT_SIGNED_INT8 = 0x08, - HIP_AD_FORMAT_SIGNED_INT16 = 0x09, - HIP_AD_FORMAT_SIGNED_INT32 = 0x0a, - HIP_AD_FORMAT_HALF = 0x10, - HIP_AD_FORMAT_FLOAT = 0x20 -}hipArray_Format; - -typedef struct HIP_ARRAY_DESCRIPTOR { - size_t Width; - size_t Height; - enum hipArray_Format Format; - unsigned int NumChannels; -}HIP_ARRAY_DESCRIPTOR; - -typedef struct HIP_ARRAY3D_DESCRIPTOR { - size_t Width; - size_t Height; - size_t Depth; - enum hipArray_Format Format; - unsigned int NumChannels; - unsigned int Flags; -}HIP_ARRAY3D_DESCRIPTOR; - -typedef struct hipArray { - void* data; // FIXME: generalize this - struct hipChannelFormatDesc desc; - unsigned int type; - unsigned int width; - unsigned int height; - unsigned int depth; - enum hipArray_Format Format; - unsigned int NumChannels; - bool isDrv; - unsigned int textureType; -}hipArray; - -#if !defined(__HIPCC_RTC__) -typedef struct hip_Memcpy2D { - size_t srcXInBytes; - size_t srcY; - hipMemoryType srcMemoryType; - const void* srcHost; - hipDeviceptr_t srcDevice; - hipArray* srcArray; - size_t srcPitch; - size_t dstXInBytes; - size_t dstY; - hipMemoryType dstMemoryType; - void* dstHost; - hipDeviceptr_t dstDevice; - hipArray* dstArray; - size_t dstPitch; - size_t WidthInBytes; - size_t Height; -} hip_Memcpy2D; -#endif // !defined(__HIPCC_RTC__) - -typedef struct hipArray* hipArray_t; -typedef hipArray_t hiparray; -typedef const struct hipArray* hipArray_const_t; - -typedef struct hipMipmappedArray { - void* data; - struct hipChannelFormatDesc desc; - unsigned int type; - unsigned int width; - unsigned int height; - unsigned int depth; - unsigned int min_mipmap_level; - unsigned int max_mipmap_level; - unsigned int flags; - enum hipArray_Format format; -} hipMipmappedArray; - -typedef struct hipMipmappedArray* hipMipmappedArray_t; - -typedef const struct hipMipmappedArray* hipMipmappedArray_const_t; - -/** - * hip resource types - */ -typedef enum hipResourceType { - hipResourceTypeArray = 0x00, - hipResourceTypeMipmappedArray = 0x01, - hipResourceTypeLinear = 0x02, - hipResourceTypePitch2D = 0x03 -}hipResourceType; - -typedef enum HIPresourcetype_enum { - HIP_RESOURCE_TYPE_ARRAY = 0x00, /**< Array resoure */ - HIP_RESOURCE_TYPE_MIPMAPPED_ARRAY = 0x01, /**< Mipmapped array resource */ - HIP_RESOURCE_TYPE_LINEAR = 0x02, /**< Linear resource */ - HIP_RESOURCE_TYPE_PITCH2D = 0x03 /**< Pitch 2D resource */ -} HIPresourcetype; - -/** - * hip address modes - */ -typedef enum HIPaddress_mode_enum { - HIP_TR_ADDRESS_MODE_WRAP = 0, - HIP_TR_ADDRESS_MODE_CLAMP = 1, - HIP_TR_ADDRESS_MODE_MIRROR = 2, - HIP_TR_ADDRESS_MODE_BORDER = 3 -} HIPaddress_mode; - -/** - * hip filter modes - */ -typedef enum HIPfilter_mode_enum { - HIP_TR_FILTER_MODE_POINT = 0, - HIP_TR_FILTER_MODE_LINEAR = 1 -} HIPfilter_mode; - -/** - * Texture descriptor - */ -typedef struct HIP_TEXTURE_DESC_st { - HIPaddress_mode addressMode[3]; /**< Address modes */ - HIPfilter_mode filterMode; /**< Filter mode */ - unsigned int flags; /**< Flags */ - unsigned int maxAnisotropy; /**< Maximum anisotropy ratio */ - HIPfilter_mode mipmapFilterMode; /**< Mipmap filter mode */ - float mipmapLevelBias; /**< Mipmap level bias */ - float minMipmapLevelClamp; /**< Mipmap minimum level clamp */ - float maxMipmapLevelClamp; /**< Mipmap maximum level clamp */ - float borderColor[4]; /**< Border Color */ - int reserved[12]; -} HIP_TEXTURE_DESC; - -/** - * hip texture resource view formats - */ -typedef enum hipResourceViewFormat { - hipResViewFormatNone = 0x00, - hipResViewFormatUnsignedChar1 = 0x01, - hipResViewFormatUnsignedChar2 = 0x02, - hipResViewFormatUnsignedChar4 = 0x03, - hipResViewFormatSignedChar1 = 0x04, - hipResViewFormatSignedChar2 = 0x05, - hipResViewFormatSignedChar4 = 0x06, - hipResViewFormatUnsignedShort1 = 0x07, - hipResViewFormatUnsignedShort2 = 0x08, - hipResViewFormatUnsignedShort4 = 0x09, - hipResViewFormatSignedShort1 = 0x0a, - hipResViewFormatSignedShort2 = 0x0b, - hipResViewFormatSignedShort4 = 0x0c, - hipResViewFormatUnsignedInt1 = 0x0d, - hipResViewFormatUnsignedInt2 = 0x0e, - hipResViewFormatUnsignedInt4 = 0x0f, - hipResViewFormatSignedInt1 = 0x10, - hipResViewFormatSignedInt2 = 0x11, - hipResViewFormatSignedInt4 = 0x12, - hipResViewFormatHalf1 = 0x13, - hipResViewFormatHalf2 = 0x14, - hipResViewFormatHalf4 = 0x15, - hipResViewFormatFloat1 = 0x16, - hipResViewFormatFloat2 = 0x17, - hipResViewFormatFloat4 = 0x18, - hipResViewFormatUnsignedBlockCompressed1 = 0x19, - hipResViewFormatUnsignedBlockCompressed2 = 0x1a, - hipResViewFormatUnsignedBlockCompressed3 = 0x1b, - hipResViewFormatUnsignedBlockCompressed4 = 0x1c, - hipResViewFormatSignedBlockCompressed4 = 0x1d, - hipResViewFormatUnsignedBlockCompressed5 = 0x1e, - hipResViewFormatSignedBlockCompressed5 = 0x1f, - hipResViewFormatUnsignedBlockCompressed6H = 0x20, - hipResViewFormatSignedBlockCompressed6H = 0x21, - hipResViewFormatUnsignedBlockCompressed7 = 0x22 -}hipResourceViewFormat; - -typedef enum HIPresourceViewFormat_enum -{ - HIP_RES_VIEW_FORMAT_NONE = 0x00, /**< No resource view format (use underlying resource format) */ - HIP_RES_VIEW_FORMAT_UINT_1X8 = 0x01, /**< 1 channel unsigned 8-bit integers */ - HIP_RES_VIEW_FORMAT_UINT_2X8 = 0x02, /**< 2 channel unsigned 8-bit integers */ - HIP_RES_VIEW_FORMAT_UINT_4X8 = 0x03, /**< 4 channel unsigned 8-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_1X8 = 0x04, /**< 1 channel signed 8-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_2X8 = 0x05, /**< 2 channel signed 8-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_4X8 = 0x06, /**< 4 channel signed 8-bit integers */ - HIP_RES_VIEW_FORMAT_UINT_1X16 = 0x07, /**< 1 channel unsigned 16-bit integers */ - HIP_RES_VIEW_FORMAT_UINT_2X16 = 0x08, /**< 2 channel unsigned 16-bit integers */ - HIP_RES_VIEW_FORMAT_UINT_4X16 = 0x09, /**< 4 channel unsigned 16-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_1X16 = 0x0a, /**< 1 channel signed 16-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_2X16 = 0x0b, /**< 2 channel signed 16-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_4X16 = 0x0c, /**< 4 channel signed 16-bit integers */ - HIP_RES_VIEW_FORMAT_UINT_1X32 = 0x0d, /**< 1 channel unsigned 32-bit integers */ - HIP_RES_VIEW_FORMAT_UINT_2X32 = 0x0e, /**< 2 channel unsigned 32-bit integers */ - HIP_RES_VIEW_FORMAT_UINT_4X32 = 0x0f, /**< 4 channel unsigned 32-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_1X32 = 0x10, /**< 1 channel signed 32-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_2X32 = 0x11, /**< 2 channel signed 32-bit integers */ - HIP_RES_VIEW_FORMAT_SINT_4X32 = 0x12, /**< 4 channel signed 32-bit integers */ - HIP_RES_VIEW_FORMAT_FLOAT_1X16 = 0x13, /**< 1 channel 16-bit floating point */ - HIP_RES_VIEW_FORMAT_FLOAT_2X16 = 0x14, /**< 2 channel 16-bit floating point */ - HIP_RES_VIEW_FORMAT_FLOAT_4X16 = 0x15, /**< 4 channel 16-bit floating point */ - HIP_RES_VIEW_FORMAT_FLOAT_1X32 = 0x16, /**< 1 channel 32-bit floating point */ - HIP_RES_VIEW_FORMAT_FLOAT_2X32 = 0x17, /**< 2 channel 32-bit floating point */ - HIP_RES_VIEW_FORMAT_FLOAT_4X32 = 0x18, /**< 4 channel 32-bit floating point */ - HIP_RES_VIEW_FORMAT_UNSIGNED_BC1 = 0x19, /**< Block compressed 1 */ - HIP_RES_VIEW_FORMAT_UNSIGNED_BC2 = 0x1a, /**< Block compressed 2 */ - HIP_RES_VIEW_FORMAT_UNSIGNED_BC3 = 0x1b, /**< Block compressed 3 */ - HIP_RES_VIEW_FORMAT_UNSIGNED_BC4 = 0x1c, /**< Block compressed 4 unsigned */ - HIP_RES_VIEW_FORMAT_SIGNED_BC4 = 0x1d, /**< Block compressed 4 signed */ - HIP_RES_VIEW_FORMAT_UNSIGNED_BC5 = 0x1e, /**< Block compressed 5 unsigned */ - HIP_RES_VIEW_FORMAT_SIGNED_BC5 = 0x1f, /**< Block compressed 5 signed */ - HIP_RES_VIEW_FORMAT_UNSIGNED_BC6H = 0x20, /**< Block compressed 6 unsigned half-float */ - HIP_RES_VIEW_FORMAT_SIGNED_BC6H = 0x21, /**< Block compressed 6 signed half-float */ - HIP_RES_VIEW_FORMAT_UNSIGNED_BC7 = 0x22 /**< Block compressed 7 */ -} HIPresourceViewFormat; - -/** - * HIP resource descriptor - */ -typedef struct hipResourceDesc { - enum hipResourceType resType; - - union { - struct { - hipArray_t array; - } array; - struct { - hipMipmappedArray_t mipmap; - } mipmap; - struct { - void* devPtr; - struct hipChannelFormatDesc desc; - size_t sizeInBytes; - } linear; - struct { - void* devPtr; - struct hipChannelFormatDesc desc; - size_t width; - size_t height; - size_t pitchInBytes; - } pitch2D; - } res; -}hipResourceDesc; - -typedef struct HIP_RESOURCE_DESC_st -{ - HIPresourcetype resType; /**< Resource type */ - - union { - struct { - hipArray_t hArray; /**< HIP array */ - } array; - struct { - hipMipmappedArray_t hMipmappedArray; /**< HIP mipmapped array */ - } mipmap; - struct { - hipDeviceptr_t devPtr; /**< Device pointer */ - hipArray_Format format; /**< Array format */ - unsigned int numChannels; /**< Channels per array element */ - size_t sizeInBytes; /**< Size in bytes */ - } linear; - struct { - hipDeviceptr_t devPtr; /**< Device pointer */ - hipArray_Format format; /**< Array format */ - unsigned int numChannels; /**< Channels per array element */ - size_t width; /**< Width of the array in elements */ - size_t height; /**< Height of the array in elements */ - size_t pitchInBytes; /**< Pitch between two rows in bytes */ - } pitch2D; - struct { - int reserved[32]; - } reserved; - } res; - - unsigned int flags; /**< Flags (must be zero) */ -} HIP_RESOURCE_DESC; - -/** - * hip resource view descriptor - */ -struct hipResourceViewDesc { - enum hipResourceViewFormat format; - size_t width; - size_t height; - size_t depth; - unsigned int firstMipmapLevel; - unsigned int lastMipmapLevel; - unsigned int firstLayer; - unsigned int lastLayer; -}; - -/** - * Resource view descriptor - */ -typedef struct HIP_RESOURCE_VIEW_DESC_st -{ - HIPresourceViewFormat format; /**< Resource view format */ - size_t width; /**< Width of the resource view */ - size_t height; /**< Height of the resource view */ - size_t depth; /**< Depth of the resource view */ - unsigned int firstMipmapLevel; /**< First defined mipmap level */ - unsigned int lastMipmapLevel; /**< Last defined mipmap level */ - unsigned int firstLayer; /**< First layer index */ - unsigned int lastLayer; /**< Last layer index */ - unsigned int reserved[16]; -} HIP_RESOURCE_VIEW_DESC; - -/** - * Memory copy types - * - */ -#if !defined(__HIPCC_RTC__) -typedef enum hipMemcpyKind { - hipMemcpyHostToHost = 0, ///< Host-to-Host Copy - hipMemcpyHostToDevice = 1, ///< Host-to-Device Copy - hipMemcpyDeviceToHost = 2, ///< Device-to-Host Copy - hipMemcpyDeviceToDevice = 3, ///< Device-to-Device Copy - hipMemcpyDefault = - 4 ///< Runtime will automatically determine copy-kind based on virtual addresses. -} hipMemcpyKind; - -typedef struct hipPitchedPtr { - void* ptr; - size_t pitch; - size_t xsize; - size_t ysize; -}hipPitchedPtr; - -typedef struct hipExtent { - size_t width; // Width in elements when referring to array memory, in bytes when referring to - // linear memory - size_t height; - size_t depth; -}hipExtent; - -typedef struct hipPos { - size_t x; - size_t y; - size_t z; -}hipPos; - -typedef struct hipMemcpy3DParms { - hipArray_t srcArray; - struct hipPos srcPos; - struct hipPitchedPtr srcPtr; - hipArray_t dstArray; - struct hipPos dstPos; - struct hipPitchedPtr dstPtr; - struct hipExtent extent; - enum hipMemcpyKind kind; -} hipMemcpy3DParms; - -typedef struct HIP_MEMCPY3D { - unsigned int srcXInBytes; - unsigned int srcY; - unsigned int srcZ; - unsigned int srcLOD; - hipMemoryType srcMemoryType; - const void* srcHost; - hipDeviceptr_t srcDevice; - hipArray_t srcArray; - unsigned int srcPitch; - unsigned int srcHeight; - unsigned int dstXInBytes; - unsigned int dstY; - unsigned int dstZ; - unsigned int dstLOD; - hipMemoryType dstMemoryType; - void* dstHost; - hipDeviceptr_t dstDevice; - hipArray_t dstArray; - unsigned int dstPitch; - unsigned int dstHeight; - unsigned int WidthInBytes; - unsigned int Height; - unsigned int Depth; -} HIP_MEMCPY3D; - -static inline struct hipPitchedPtr make_hipPitchedPtr(void* d, size_t p, size_t xsz, - size_t ysz) { - struct hipPitchedPtr s; - - s.ptr = d; - s.pitch = p; - s.xsize = xsz; - s.ysize = ysz; - - return s; -} - -static inline struct hipPos make_hipPos(size_t x, size_t y, size_t z) { - struct hipPos p; - - p.x = x; - p.y = y; - p.z = z; - - return p; -} - -static inline struct hipExtent make_hipExtent(size_t w, size_t h, size_t d) { - struct hipExtent e; - - e.width = w; - e.height = h; - e.depth = d; - - return e; -} - -typedef enum hipFunction_attribute { - HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, - HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, - HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES, - HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, - HIP_FUNC_ATTRIBUTE_NUM_REGS, - HIP_FUNC_ATTRIBUTE_PTX_VERSION, - HIP_FUNC_ATTRIBUTE_BINARY_VERSION, - HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA, - HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, - HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, - HIP_FUNC_ATTRIBUTE_MAX -}hipFunction_attribute; -#endif // !defined(__HIPCC_RTC__) -#endif diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_atomic.h b/src/hipamd/include/hip/amd_detail/amd_hip_atomic.h deleted file mode 100644 index 6a25533beb..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_atomic.h +++ /dev/null @@ -1,691 +0,0 @@ - - -#include "amd_device_functions.h" - -#if __has_builtin(__hip_atomic_compare_exchange_strong) - -#if !__HIP_DEVICE_COMPILE__ -//TODO: Remove this after compiler pre-defines the following Macros. -#define __HIP_MEMORY_SCOPE_SINGLETHREAD 1 -#define __HIP_MEMORY_SCOPE_WAVEFRONT 2 -#define __HIP_MEMORY_SCOPE_WORKGROUP 3 -#define __HIP_MEMORY_SCOPE_AGENT 4 -#define __HIP_MEMORY_SCOPE_SYSTEM 5 -#endif - -__device__ -inline -int atomicCAS(int* address, int compare, int val) { - __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT); - return compare; -} - -__device__ -inline -int atomicCAS_system(int* address, int compare, int val) { - __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_SYSTEM); - return compare; -} - -__device__ -inline -unsigned int atomicCAS(unsigned int* address, unsigned int compare, unsigned int val) { - __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT); - return compare; -} - -__device__ -inline -unsigned int atomicCAS_system(unsigned int* address, unsigned int compare, unsigned int val) { - __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_SYSTEM); - return compare; -} - -__device__ -inline -unsigned long long atomicCAS(unsigned long long* address, unsigned long long compare, - unsigned long long val) { - __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT); - return compare; -} - -__device__ -inline -unsigned long long atomicCAS_system(unsigned long long* address, unsigned long long compare, - unsigned long long val) { - __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_SYSTEM); - return compare; -} - -__device__ -inline -int atomicAdd(int* address, int val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -int atomicAdd_system(int* address, int val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicAdd(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned int atomicAdd_system(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned long long atomicAdd(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned long long atomicAdd_system(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -float atomicAdd(float* address, float val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -float atomicAdd_system(float* address, float val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -#if !defined(__HIPCC_RTC__) -DEPRECATED("use atomicAdd instead") -#endif // !defined(__HIPCC_RTC__) -__device__ -inline -void atomicAddNoRet(float* address, float val) -{ - __ockl_atomic_add_noret_f32(address, val); -} - -__device__ -inline -double atomicAdd(double* address, double val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -double atomicAdd_system(double* address, double val) { - return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -int atomicSub(int* address, int val) { - return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -int atomicSub_system(int* address, int val) { - return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicSub(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned int atomicSub_system(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -int atomicExch(int* address, int val) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -int atomicExch_system(int* address, int val) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicExch(unsigned int* address, unsigned int val) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned int atomicExch_system(unsigned int* address, unsigned int val) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned long long atomicExch(unsigned long long* address, unsigned long long val) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned long long atomicExch_system(unsigned long long* address, unsigned long long val) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -float atomicExch(float* address, float val) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -float atomicExch_system(float* address, float val) { - return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -int atomicMin(int* address, int val) { - return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -int atomicMin_system(int* address, int val) { - return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicMin(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned int atomicMin_system(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned long long atomicMin(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned long long atomicMin_system(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -int atomicMax(int* address, int val) { - return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -int atomicMax_system(int* address, int val) { - return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicMax(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned int atomicMax_system(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned long long atomicMax(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned long long atomicMax_system(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicInc(unsigned int* address, unsigned int val) -{ - __device__ - extern - unsigned int __builtin_amdgcn_atomic_inc( - unsigned int*, - unsigned int, - unsigned int, - unsigned int, - bool) __asm("llvm.amdgcn.atomic.inc.i32.p0i32"); - - return __builtin_amdgcn_atomic_inc( - address, val, __ATOMIC_RELAXED, 1 /* Device scope */, false); -} - -__device__ -inline -unsigned int atomicDec(unsigned int* address, unsigned int val) -{ - __device__ - extern - unsigned int __builtin_amdgcn_atomic_dec( - unsigned int*, - unsigned int, - unsigned int, - unsigned int, - bool) __asm("llvm.amdgcn.atomic.dec.i32.p0i32"); - - return __builtin_amdgcn_atomic_dec( - address, val, __ATOMIC_RELAXED, 1 /* Device scope */, false); -} - -__device__ -inline -int atomicAnd(int* address, int val) { - return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -int atomicAnd_system(int* address, int val) { - return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicAnd(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned int atomicAnd_system(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} -__device__ -inline -unsigned long long atomicAnd(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned long long atomicAnd_system(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -int atomicOr(int* address, int val) { - return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -int atomicOr_system(int* address, int val) { - return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicOr(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned int atomicOr_system(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned long long atomicOr(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned long long atomicOr_system(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -int atomicXor(int* address, int val) { - return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -int atomicXor_system(int* address, int val) { - return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned int atomicXor(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned int atomicXor_system(unsigned int* address, unsigned int val) { - return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -__device__ -inline -unsigned long long atomicXor(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); -} - -__device__ -inline -unsigned long long atomicXor_system(unsigned long long* address, unsigned long long val) { - return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); -} - -#else - -__device__ -inline -int atomicCAS(int* address, int compare, int val) -{ - __atomic_compare_exchange_n( - address, &compare, val, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); - - return compare; -} -__device__ -inline -unsigned int atomicCAS( - unsigned int* address, unsigned int compare, unsigned int val) -{ - __atomic_compare_exchange_n( - address, &compare, val, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); - - return compare; -} -__device__ -inline -unsigned long long atomicCAS( - unsigned long long* address, - unsigned long long compare, - unsigned long long val) -{ - __atomic_compare_exchange_n( - address, &compare, val, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); - - return compare; -} - -__device__ -inline -int atomicAdd(int* address, int val) -{ - return __atomic_fetch_add(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned int atomicAdd(unsigned int* address, unsigned int val) -{ - return __atomic_fetch_add(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned long long atomicAdd( - unsigned long long* address, unsigned long long val) -{ - return __atomic_fetch_add(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -float atomicAdd(float* address, float val) -{ - return __atomic_fetch_add(address, val, __ATOMIC_RELAXED); -} - -#if !defined(__HIPCC_RTC__) -DEPRECATED("use atomicAdd instead") -#endif // !defined(__HIPCC_RTC__) -__device__ -inline -void atomicAddNoRet(float* address, float val) -{ - __ockl_atomic_add_noret_f32(address, val); -} - -__device__ -inline -double atomicAdd(double* address, double val) -{ - return __atomic_fetch_add(address, val, __ATOMIC_RELAXED); -} - -__device__ -inline -int atomicSub(int* address, int val) -{ - return __atomic_fetch_sub(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned int atomicSub(unsigned int* address, unsigned int val) -{ - return __atomic_fetch_sub(address, val, __ATOMIC_RELAXED); -} - -__device__ -inline -int atomicExch(int* address, int val) -{ - return __atomic_exchange_n(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned int atomicExch(unsigned int* address, unsigned int val) -{ - return __atomic_exchange_n(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned long long atomicExch(unsigned long long* address, unsigned long long val) -{ - return __atomic_exchange_n(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -float atomicExch(float* address, float val) -{ - return __uint_as_float(__atomic_exchange_n( - reinterpret_cast(address), - __float_as_uint(val), - __ATOMIC_RELAXED)); -} - -__device__ -inline -int atomicMin(int* address, int val) -{ - return __atomic_fetch_min(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned int atomicMin(unsigned int* address, unsigned int val) -{ - return __atomic_fetch_min(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned long long atomicMin( - unsigned long long* address, unsigned long long val) -{ - unsigned long long tmp{__atomic_load_n(address, __ATOMIC_RELAXED)}; - while (val < tmp) { - const auto tmp1 = __atomic_load_n(address, __ATOMIC_RELAXED); - - if (tmp1 != tmp) { tmp = tmp1; continue; } - - tmp = atomicCAS(address, tmp, val); - } - - return tmp; -} - -__device__ -inline -int atomicMax(int* address, int val) -{ - return __atomic_fetch_max(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned int atomicMax(unsigned int* address, unsigned int val) -{ - return __atomic_fetch_max(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned long long atomicMax( - unsigned long long* address, unsigned long long val) -{ - unsigned long long tmp{__atomic_load_n(address, __ATOMIC_RELAXED)}; - while (tmp < val) { - const auto tmp1 = __atomic_load_n(address, __ATOMIC_RELAXED); - - if (tmp1 != tmp) { tmp = tmp1; continue; } - - tmp = atomicCAS(address, tmp, val); - } - - return tmp; -} - -__device__ -inline -unsigned int atomicInc(unsigned int* address, unsigned int val) -{ - __device__ - extern - unsigned int __builtin_amdgcn_atomic_inc( - unsigned int*, - unsigned int, - unsigned int, - unsigned int, - bool) __asm("llvm.amdgcn.atomic.inc.i32.p0i32"); - - return __builtin_amdgcn_atomic_inc( - address, val, __ATOMIC_RELAXED, 1 /* Device scope */, false); -} - -__device__ -inline -unsigned int atomicDec(unsigned int* address, unsigned int val) -{ - __device__ - extern - unsigned int __builtin_amdgcn_atomic_dec( - unsigned int*, - unsigned int, - unsigned int, - unsigned int, - bool) __asm("llvm.amdgcn.atomic.dec.i32.p0i32"); - - return __builtin_amdgcn_atomic_dec( - address, val, __ATOMIC_RELAXED, 1 /* Device scope */, false); -} - -__device__ -inline -int atomicAnd(int* address, int val) -{ - return __atomic_fetch_and(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned int atomicAnd(unsigned int* address, unsigned int val) -{ - return __atomic_fetch_and(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned long long atomicAnd( - unsigned long long* address, unsigned long long val) -{ - return __atomic_fetch_and(address, val, __ATOMIC_RELAXED); -} - -__device__ -inline -int atomicOr(int* address, int val) -{ - return __atomic_fetch_or(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned int atomicOr(unsigned int* address, unsigned int val) -{ - return __atomic_fetch_or(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned long long atomicOr( - unsigned long long* address, unsigned long long val) -{ - return __atomic_fetch_or(address, val, __ATOMIC_RELAXED); -} - -__device__ -inline -int atomicXor(int* address, int val) -{ - return __atomic_fetch_xor(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned int atomicXor(unsigned int* address, unsigned int val) -{ - return __atomic_fetch_xor(address, val, __ATOMIC_RELAXED); -} -__device__ -inline -unsigned long long atomicXor( - unsigned long long* address, unsigned long long val) -{ - return __atomic_fetch_xor(address, val, __ATOMIC_RELAXED); -} - -#endif diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_common.h b/src/hipamd/include/hip/amd_detail/amd_hip_common.h deleted file mode 100644 index 4881ade678..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_common.h +++ /dev/null @@ -1,32 +0,0 @@ -/* -Copyright (c) 2019 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H - -#if defined(__clang__) && defined(__HIP__) -#define __HIP_CLANG_ONLY__ 1 -#else -#define __HIP_CLANG_ONLY__ 0 -#endif - -#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_complex.h b/src/hipamd/include/hip/amd_detail/amd_hip_complex.h deleted file mode 100644 index 214172cd5e..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_complex.h +++ /dev/null @@ -1,309 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H - -#include "hip/amd_detail/amd_hip_vector_types.h" - -#if defined(__HIPCC_RTC__) -#define __HOST_DEVICE__ __device__ -#else -#define __HOST_DEVICE__ __host__ __device__ -// TODO: Clang has a bug which allows device functions to call std functions -// when std functions are introduced into default namespace by using statement. -// math.h may be included after this bug is fixed. -#if __cplusplus -#include -#else -#include "math.h" -#endif -#endif // !defined(__HIPCC_RTC__) - -#if __cplusplus -#define COMPLEX_NEG_OP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type operator-(const type& op) { \ - type ret; \ - ret.x = -op.x; \ - ret.y = -op.y; \ - return ret; \ - } - -#define COMPLEX_EQ_OP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline bool operator==(const type& lhs, const type& rhs) { \ - return lhs.x == rhs.x && lhs.y == rhs.y; \ - } - -#define COMPLEX_NE_OP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline bool operator!=(const type& lhs, const type& rhs) { \ - return !(lhs == rhs); \ - } - -#define COMPLEX_ADD_OP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type operator+(const type& lhs, const type& rhs) { \ - type ret; \ - ret.x = lhs.x + rhs.x; \ - ret.y = lhs.y + rhs.y; \ - return ret; \ - } - -#define COMPLEX_SUB_OP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type operator-(const type& lhs, const type& rhs) { \ - type ret; \ - ret.x = lhs.x - rhs.x; \ - ret.y = lhs.y - rhs.y; \ - return ret; \ - } - -#define COMPLEX_MUL_OP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type operator*(const type& lhs, const type& rhs) { \ - type ret; \ - ret.x = lhs.x * rhs.x - lhs.y * rhs.y; \ - ret.y = lhs.x * rhs.y + lhs.y * rhs.x; \ - return ret; \ - } - -#define COMPLEX_DIV_OP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type operator/(const type& lhs, const type& rhs) { \ - type ret; \ - ret.x = (lhs.x * rhs.x + lhs.y * rhs.y); \ - ret.y = (rhs.x * lhs.y - lhs.x * rhs.y); \ - ret.x = ret.x / (rhs.x * rhs.x + rhs.y * rhs.y); \ - ret.y = ret.y / (rhs.x * rhs.x + rhs.y * rhs.y); \ - return ret; \ - } - -#define COMPLEX_ADD_PREOP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type& operator+=(type& lhs, const type& rhs) { \ - lhs.x += rhs.x; \ - lhs.y += rhs.y; \ - return lhs; \ - } - -#define COMPLEX_SUB_PREOP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type& operator-=(type& lhs, const type& rhs) { \ - lhs.x -= rhs.x; \ - lhs.y -= rhs.y; \ - return lhs; \ - } - -#define COMPLEX_MUL_PREOP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type& operator*=(type& lhs, const type& rhs) { \ - lhs = lhs * rhs; \ - return lhs; \ - } - -#define COMPLEX_DIV_PREOP_OVERLOAD(type) \ - __HOST_DEVICE__ static inline type& operator/=(type& lhs, const type& rhs) { \ - lhs = lhs / rhs; \ - return lhs; \ - } - -#define COMPLEX_SCALAR_PRODUCT(type, type1) \ - __HOST_DEVICE__ static inline type operator*(const type& lhs, type1 rhs) { \ - type ret; \ - ret.x = lhs.x * rhs; \ - ret.y = lhs.y * rhs; \ - return ret; \ - } - -#endif - -typedef float2 hipFloatComplex; - -__HOST_DEVICE__ static inline float hipCrealf(hipFloatComplex z) { return z.x; } - -__HOST_DEVICE__ static inline float hipCimagf(hipFloatComplex z) { return z.y; } - -__HOST_DEVICE__ static inline hipFloatComplex make_hipFloatComplex(float a, float b) { - hipFloatComplex z; - z.x = a; - z.y = b; - return z; -} - -__HOST_DEVICE__ static inline hipFloatComplex hipConjf(hipFloatComplex z) { - hipFloatComplex ret; - ret.x = z.x; - ret.y = -z.y; - return ret; -} - -__HOST_DEVICE__ static inline float hipCsqabsf(hipFloatComplex z) { - return z.x * z.x + z.y * z.y; -} - -__HOST_DEVICE__ static inline hipFloatComplex hipCaddf(hipFloatComplex p, hipFloatComplex q) { - return make_hipFloatComplex(p.x + q.x, p.y + q.y); -} - -__HOST_DEVICE__ static inline hipFloatComplex hipCsubf(hipFloatComplex p, hipFloatComplex q) { - return make_hipFloatComplex(p.x - q.x, p.y - q.y); -} - -__HOST_DEVICE__ static inline hipFloatComplex hipCmulf(hipFloatComplex p, hipFloatComplex q) { - return make_hipFloatComplex(p.x * q.x - p.y * q.y, p.y * q.x + p.x * q.y); -} - -__HOST_DEVICE__ static inline hipFloatComplex hipCdivf(hipFloatComplex p, hipFloatComplex q) { - float sqabs = hipCsqabsf(q); - hipFloatComplex ret; - ret.x = (p.x * q.x + p.y * q.y) / sqabs; - ret.y = (p.y * q.x - p.x * q.y) / sqabs; - return ret; -} - -__HOST_DEVICE__ static inline float hipCabsf(hipFloatComplex z) { return sqrtf(hipCsqabsf(z)); } - - -typedef double2 hipDoubleComplex; - -__HOST_DEVICE__ static inline double hipCreal(hipDoubleComplex z) { return z.x; } - -__HOST_DEVICE__ static inline double hipCimag(hipDoubleComplex z) { return z.y; } - -__HOST_DEVICE__ static inline hipDoubleComplex make_hipDoubleComplex(double a, double b) { - hipDoubleComplex z; - z.x = a; - z.y = b; - return z; -} - -__HOST_DEVICE__ static inline hipDoubleComplex hipConj(hipDoubleComplex z) { - hipDoubleComplex ret; - ret.x = z.x; - ret.y = -z.y; - return ret; -} - -__HOST_DEVICE__ static inline double hipCsqabs(hipDoubleComplex z) { - return z.x * z.x + z.y * z.y; -} - -__HOST_DEVICE__ static inline hipDoubleComplex hipCadd(hipDoubleComplex p, hipDoubleComplex q) { - return make_hipDoubleComplex(p.x + q.x, p.y + q.y); -} - -__HOST_DEVICE__ static inline hipDoubleComplex hipCsub(hipDoubleComplex p, hipDoubleComplex q) { - return make_hipDoubleComplex(p.x - q.x, p.y - q.y); -} - -__HOST_DEVICE__ static inline hipDoubleComplex hipCmul(hipDoubleComplex p, hipDoubleComplex q) { - return make_hipDoubleComplex(p.x * q.x - p.y * q.y, p.y * q.x + p.x * q.y); -} - -__HOST_DEVICE__ static inline hipDoubleComplex hipCdiv(hipDoubleComplex p, hipDoubleComplex q) { - double sqabs = hipCsqabs(q); - hipDoubleComplex ret; - ret.x = (p.x * q.x + p.y * q.y) / sqabs; - ret.y = (p.y * q.x - p.x * q.y) / sqabs; - return ret; -} - -__HOST_DEVICE__ static inline double hipCabs(hipDoubleComplex z) { return sqrt(hipCsqabs(z)); } - - -#if __cplusplus - -COMPLEX_NEG_OP_OVERLOAD(hipFloatComplex) -COMPLEX_EQ_OP_OVERLOAD(hipFloatComplex) -COMPLEX_NE_OP_OVERLOAD(hipFloatComplex) -COMPLEX_ADD_OP_OVERLOAD(hipFloatComplex) -COMPLEX_SUB_OP_OVERLOAD(hipFloatComplex) -COMPLEX_MUL_OP_OVERLOAD(hipFloatComplex) -COMPLEX_DIV_OP_OVERLOAD(hipFloatComplex) -COMPLEX_ADD_PREOP_OVERLOAD(hipFloatComplex) -COMPLEX_SUB_PREOP_OVERLOAD(hipFloatComplex) -COMPLEX_MUL_PREOP_OVERLOAD(hipFloatComplex) -COMPLEX_DIV_PREOP_OVERLOAD(hipFloatComplex) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, unsigned short) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, signed short) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, unsigned int) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, signed int) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, float) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, unsigned long) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, signed long) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, double) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, signed long long) -COMPLEX_SCALAR_PRODUCT(hipFloatComplex, unsigned long long) - -COMPLEX_NEG_OP_OVERLOAD(hipDoubleComplex) -COMPLEX_EQ_OP_OVERLOAD(hipDoubleComplex) -COMPLEX_NE_OP_OVERLOAD(hipDoubleComplex) -COMPLEX_ADD_OP_OVERLOAD(hipDoubleComplex) -COMPLEX_SUB_OP_OVERLOAD(hipDoubleComplex) -COMPLEX_MUL_OP_OVERLOAD(hipDoubleComplex) -COMPLEX_DIV_OP_OVERLOAD(hipDoubleComplex) -COMPLEX_ADD_PREOP_OVERLOAD(hipDoubleComplex) -COMPLEX_SUB_PREOP_OVERLOAD(hipDoubleComplex) -COMPLEX_MUL_PREOP_OVERLOAD(hipDoubleComplex) -COMPLEX_DIV_PREOP_OVERLOAD(hipDoubleComplex) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, unsigned short) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, signed short) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, unsigned int) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, signed int) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, float) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, unsigned long) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, signed long) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, double) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, signed long long) -COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, unsigned long long) - -#endif - - -typedef hipFloatComplex hipComplex; - -__HOST_DEVICE__ static inline hipComplex make_hipComplex(float x, float y) { - return make_hipFloatComplex(x, y); -} - -__HOST_DEVICE__ static inline hipFloatComplex hipComplexDoubleToFloat(hipDoubleComplex z) { - return make_hipFloatComplex((float)z.x, (float)z.y); -} - -__HOST_DEVICE__ static inline hipDoubleComplex hipComplexFloatToDouble(hipFloatComplex z) { - return make_hipDoubleComplex((double)z.x, (double)z.y); -} - -__HOST_DEVICE__ static inline hipComplex hipCfmaf(hipComplex p, hipComplex q, hipComplex r) { - float real = (p.x * q.x) + r.x; - float imag = (q.x * p.y) + r.y; - - real = -(p.y * q.y) + real; - imag = (p.x * q.y) + imag; - - return make_hipComplex(real, imag); -} - -__HOST_DEVICE__ static inline hipDoubleComplex hipCfma(hipDoubleComplex p, hipDoubleComplex q, - hipDoubleComplex r) { - double real = (p.x * q.x) + r.x; - double imag = (q.x * p.y) + r.y; - - real = -(p.y * q.y) + real; - imag = (p.x * q.y) + imag; - - return make_hipDoubleComplex(real, imag); -} - -#endif //HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h b/src/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h deleted file mode 100644 index cd3e4bf085..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h +++ /dev/null @@ -1,510 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/hip_cooperative_groups.h - * - * @brief Device side implementation of `Cooperative Group` feature. - * - * Defines new types and device API wrappers related to `Cooperative Group` - * feature, which the programmer can directly use in his kernel(s) in order to - * make use of this feature. - */ -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_H - -#if __cplusplus -#include - -namespace cooperative_groups { - -/** \brief The base type of all cooperative group types - * - * \details Holds the key properties of a constructed cooperative group types - * object, like the group type, its size, etc - */ -class thread_group { - protected: - uint32_t _type; // thread_group type - uint32_t _size; // total number of threads in the tread_group - uint64_t _mask; // Lanemask for coalesced and tiled partitioned group types, - // LSB represents lane 0, and MSB represents lane 63 - - // Construct a thread group, and set thread group type and other essential - // thread group properties. This generic thread group is directly constructed - // only when the group is supposed to contain only the calling the thread - // (throurh the API - `this_thread()`), and in all other cases, this thread - // group object is a sub-object of some other derived thread group object - __CG_QUALIFIER__ thread_group(internal::group_type type, uint32_t size, - uint64_t mask = (uint64_t)0) { - _type = type; - _size = size; - _mask = mask; - } - - struct _tiled_info { - bool is_tiled; - unsigned int size; - } tiled_info; - - friend __CG_QUALIFIER__ thread_group tiled_partition(const thread_group& parent, - unsigned int tile_size); - friend class thread_block; - - public: - // Total number of threads in the thread group, and this serves the purpose - // for all derived cooperative group types since their `size` is directly - // saved during the construction - __CG_QUALIFIER__ uint32_t size() const { return _size; } - __CG_QUALIFIER__ unsigned int cg_type() const { return _type; } - // Rank of the calling thread within [0, size()) - __CG_QUALIFIER__ uint32_t thread_rank() const; - // Is this cooperative group type valid? - __CG_QUALIFIER__ bool is_valid() const; - // synchronize the threads in the thread group - __CG_QUALIFIER__ void sync() const; -}; - -/** \brief The multi-grid cooperative group type - * - * \details Represents an inter-device cooperative group type where the - * participating threads within the group spans across multple - * devices, running the (same) kernel on these devices - */ -class multi_grid_group : public thread_group { - // Only these friend functions are allowed to construct an object of this class - // and access its resources - friend __CG_QUALIFIER__ multi_grid_group this_multi_grid(); - - protected: - // Construct mutli-grid thread group (through the API this_multi_grid()) - explicit __CG_QUALIFIER__ multi_grid_group(uint32_t size) - : thread_group(internal::cg_multi_grid, size) {} - - public: - // Number of invocations participating in this multi-grid group. In other - // words, the number of GPUs - __CG_QUALIFIER__ uint32_t num_grids() { return internal::multi_grid::num_grids(); } - // Rank of this invocation. In other words, an ID number within the range - // [0, num_grids()) of the GPU, this kernel is running on - __CG_QUALIFIER__ uint32_t grid_rank() { return internal::multi_grid::grid_rank(); } - __CG_QUALIFIER__ uint32_t thread_rank() const { return internal::multi_grid::thread_rank(); } - __CG_QUALIFIER__ bool is_valid() const { return internal::multi_grid::is_valid(); } - __CG_QUALIFIER__ void sync() const { internal::multi_grid::sync(); } -}; - -/** \brief User exposed API interface to construct multi-grid cooperative - * group type object - `multi_grid_group` - * - * \details User is not allowed to directly construct an object of type - * `multi_grid_group`. Instead, he should construct it through this - * API function - */ -__CG_QUALIFIER__ multi_grid_group this_multi_grid() { - return multi_grid_group(internal::multi_grid::size()); -} - -/** \brief The grid cooperative group type - * - * \details Represents an inter-workgroup cooperative group type where the - * participating threads within the group spans across multiple - * workgroups running the (same) kernel on the same device - */ -class grid_group : public thread_group { - // Only these friend functions are allowed to construct an object of this class - // and access its resources - friend __CG_QUALIFIER__ grid_group this_grid(); - - protected: - // Construct grid thread group (through the API this_grid()) - explicit __CG_QUALIFIER__ grid_group(uint32_t size) : thread_group(internal::cg_grid, size) {} - - public: - __CG_QUALIFIER__ uint32_t thread_rank() const { return internal::grid::thread_rank(); } - __CG_QUALIFIER__ bool is_valid() const { return internal::grid::is_valid(); } - __CG_QUALIFIER__ void sync() const { internal::grid::sync(); } -}; - -/** \brief User exposed API interface to construct grid cooperative group type - * object - `grid_group` - * - * \details User is not allowed to directly construct an object of type - * `multi_grid_group`. Instead, he should construct it through this - * API function - */ -__CG_QUALIFIER__ grid_group this_grid() { return grid_group(internal::grid::size()); } - -/** \brief The workgroup (thread-block in CUDA terminology) cooperative group - * type - * - * \details Represents an intra-workgroup cooperative group type where the - * participating threads within the group are exactly the same threads - * which are participated in the currently executing `workgroup` - */ -class thread_block : public thread_group { - // Only these friend functions are allowed to construct an object of thi - // class and access its resources - friend __CG_QUALIFIER__ thread_block this_thread_block(); - friend __CG_QUALIFIER__ thread_group tiled_partition(const thread_group& parent, - unsigned int tile_size); - friend __CG_QUALIFIER__ thread_group tiled_partition(const thread_block& parent, - unsigned int tile_size); - - protected: - // Construct a workgroup thread group (through the API this_thread_block()) - explicit __CG_QUALIFIER__ thread_block(uint32_t size) - : thread_group(internal::cg_workgroup, size) {} - - __CG_QUALIFIER__ thread_group new_tiled_group(unsigned int tile_size) const { - const bool pow2 = ((tile_size & (tile_size - 1)) == 0); - // Invalid tile size, assert - if (!tile_size || (tile_size > WAVEFRONT_SIZE) || !pow2) { - assert(false && "invalid tile size"); - } - - thread_group tiledGroup = thread_group(internal::cg_tiled_group, tile_size); - tiledGroup.tiled_info.size = tile_size; - tiledGroup.tiled_info.is_tiled = true; - return tiledGroup; - } - - public: - // 3-dimensional block index within the grid - __CG_QUALIFIER__ dim3 group_index() { return internal::workgroup::group_index(); } - // 3-dimensional thread index within the block - __CG_QUALIFIER__ dim3 thread_index() { return internal::workgroup::thread_index(); } - __CG_QUALIFIER__ uint32_t thread_rank() const { return internal::workgroup::thread_rank(); } - __CG_QUALIFIER__ bool is_valid() const { return internal::workgroup::is_valid(); } - __CG_QUALIFIER__ void sync() const { internal::workgroup::sync(); } -}; - -/** \brief User exposed API interface to construct workgroup cooperative - * group type object - `thread_block`. - * - * \details User is not allowed to directly construct an object of type - * `thread_block`. Instead, he should construct it through this API - * function. - */ -__CG_QUALIFIER__ thread_block this_thread_block() { - return thread_block(internal::workgroup::size()); -} - -/** \brief The tiled_group cooperative group type - * - * \details Represents one tiled thread group in a wavefront. - * This group type also supports sub-wave level intrinsics. - */ - -class tiled_group : public thread_group { - private: - friend __CG_QUALIFIER__ thread_group tiled_partition(const thread_group& parent, - unsigned int tile_size); - friend __CG_QUALIFIER__ tiled_group tiled_partition(const tiled_group& parent, - unsigned int tile_size); - - __CG_QUALIFIER__ tiled_group new_tiled_group(unsigned int tile_size) const { - const bool pow2 = ((tile_size & (tile_size - 1)) == 0); - - if (!tile_size || (tile_size > WAVEFRONT_SIZE) || !pow2) { - assert(false && "invalid tile size"); - } - - if (size() <= tile_size) { - return (*this); - } - - tiled_group tiledGroup = tiled_group(tile_size); - tiledGroup.tiled_info.is_tiled = true; - return tiledGroup; - } - - protected: - explicit __CG_QUALIFIER__ tiled_group(unsigned int tileSize) - : thread_group(internal::cg_tiled_group, tileSize) { - tiled_info.size = tileSize; - tiled_info.is_tiled = true; - } - - public: - __CG_QUALIFIER__ unsigned int size() const { return (tiled_info.size); } - - __CG_QUALIFIER__ unsigned int thread_rank() const { - return (internal::workgroup::thread_rank() & (tiled_info.size - 1)); - } - - __CG_QUALIFIER__ void sync() const { - // enforce memory ordering for memory instructions. - __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "agent"); - } -}; - -/** - * Implemenation of all publicly exposed base class APIs - */ -__CG_QUALIFIER__ uint32_t thread_group::thread_rank() const { - switch (this->_type) { - case internal::cg_multi_grid: { - return (static_cast(this)->thread_rank()); - } - case internal::cg_grid: { - return (static_cast(this)->thread_rank()); - } - case internal::cg_workgroup: { - return (static_cast(this)->thread_rank()); - } - case internal::cg_tiled_group: { - return (static_cast(this)->thread_rank()); - } - default: { - assert(false && "invalid cooperative group type"); - return -1; - } - } -} - -__CG_QUALIFIER__ bool thread_group::is_valid() const { - switch (this->_type) { - case internal::cg_multi_grid: { - return (static_cast(this)->is_valid()); - } - case internal::cg_grid: { - return (static_cast(this)->is_valid()); - } - case internal::cg_workgroup: { - return (static_cast(this)->is_valid()); - } - case internal::cg_tiled_group: { - return (static_cast(this)->is_valid()); - } - default: { - assert(false && "invalid cooperative group type"); - return false; - } - } -} - -__CG_QUALIFIER__ void thread_group::sync() const { - switch (this->_type) { - case internal::cg_multi_grid: { - static_cast(this)->sync(); - break; - } - case internal::cg_grid: { - static_cast(this)->sync(); - break; - } - case internal::cg_workgroup: { - static_cast(this)->sync(); - break; - } - case internal::cg_tiled_group: { - static_cast(this)->sync(); - break; - } - default: { - assert(false && "invalid cooperative group type"); - } - } -} - -/** - * Implemenation of publicly exposed `wrapper` APIs on top of basic cooperative - * group type APIs - */ -template __CG_QUALIFIER__ uint32_t group_size(CGTy const& g) { return g.size(); } - -template __CG_QUALIFIER__ uint32_t thread_rank(CGTy const& g) { - return g.thread_rank(); -} - -template __CG_QUALIFIER__ bool is_valid(CGTy const& g) { return g.is_valid(); } - -template __CG_QUALIFIER__ void sync(CGTy const& g) { g.sync(); } - -template class tile_base { - protected: - _CG_STATIC_CONST_DECL_ unsigned int numThreads = tileSize; - - public: - // Rank of the thread within this tile - _CG_STATIC_CONST_DECL_ unsigned int thread_rank() { - return (internal::workgroup::thread_rank() & (numThreads - 1)); - } - - // Number of threads within this tile - __CG_STATIC_QUALIFIER__ unsigned int size() { return numThreads; } -}; - -template class thread_block_tile_base : public tile_base { - static_assert(is_valid_tile_size::value, - "Tile size is either not a power of 2 or greater than the wavefront size"); - using tile_base::numThreads; - - public: - __CG_STATIC_QUALIFIER__ void sync() { - // enforce ordering for memory instructions - __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "agent"); - } - - template __CG_QUALIFIER__ T shfl(T var, int srcRank) const { - static_assert(is_valid_type::value, "Neither an integer or float type."); - return (__shfl(var, srcRank, numThreads)); - } - - template __CG_QUALIFIER__ T shfl_down(T var, unsigned int lane_delta) const { - static_assert(is_valid_type::value, "Neither an integer or float type."); - return (__shfl_down(var, lane_delta, numThreads)); - } - - template __CG_QUALIFIER__ T shfl_up(T var, unsigned int lane_delta) const { - static_assert(is_valid_type::value, "Neither an integer or float type."); - return (__shfl_up(var, lane_delta, numThreads)); - } - - template __CG_QUALIFIER__ T shfl_xor(T var, unsigned int laneMask) const { - static_assert(is_valid_type::value, "Neither an integer or float type."); - return (__shfl_xor(var, laneMask, numThreads)); - } -}; - -/** \brief Group type - thread_block_tile - * - * \details Represents one tile of thread group. - */ - -template -class thread_block_tile_type : public thread_block_tile_base, public tiled_group { - _CG_STATIC_CONST_DECL_ unsigned int numThreads = tileSize; - - friend class thread_block_tile_type; - - typedef thread_block_tile_base tbtBase; - - protected: - __CG_QUALIFIER__ thread_block_tile_type() : tiled_group(numThreads) { - tiled_info.size = numThreads; - tiled_info.is_tiled = true; - } - - public: - using tbtBase::size; - using tbtBase::sync; - using tbtBase::thread_rank; -}; - - -/** \brief User exposed API to partition groups. - * - * \details A collective operation that partitions the parent group into a one-dimensional, - * row-major, tiling of subgroups. - */ - -__CG_QUALIFIER__ thread_group tiled_partition(const thread_group& parent, unsigned int tile_size) { - if (parent.cg_type() == internal::cg_tiled_group) { - const tiled_group* cg = static_cast(&parent); - return cg->new_tiled_group(tile_size); - } else { - const thread_block* tb = static_cast(&parent); - return tb->new_tiled_group(tile_size); - } -} - -// Thread block type overload -__CG_QUALIFIER__ thread_group tiled_partition(const thread_block& parent, unsigned int tile_size) { - return (parent.new_tiled_group(tile_size)); -} - -// Coalesced group type overload -__CG_QUALIFIER__ tiled_group tiled_partition(const tiled_group& parent, unsigned int tile_size) { - return (parent.new_tiled_group(tile_size)); -} - -template class thread_block_tile; - -namespace impl { -template class thread_block_tile_internal; - -template -class thread_block_tile_internal : public thread_block_tile_type { - protected: - template - __CG_QUALIFIER__ thread_block_tile_internal( - const thread_block_tile_internal& g) - : thread_block_tile_type() {} - - __CG_QUALIFIER__ thread_block_tile_internal(const thread_block& g) - : thread_block_tile_type() {} -}; -} // namespace impl - -template -class thread_block_tile : public impl::thread_block_tile_internal { - protected: - __CG_QUALIFIER__ thread_block_tile(const ParentCGTy& g) - : impl::thread_block_tile_internal(g) {} - - public: - __CG_QUALIFIER__ operator thread_block_tile() const { - return thread_block_tile(*this); - } -}; - - -template -class thread_block_tile : public impl::thread_block_tile_internal { - template friend class thread_block_tile; - - protected: - public: - template - __CG_QUALIFIER__ thread_block_tile(const thread_block_tile& g) - : impl::thread_block_tile_internal(g) {} -}; - -template class thread_block_tile; - -namespace impl { -template struct tiled_partition_internal; - -template -struct tiled_partition_internal : public thread_block_tile { - __CG_QUALIFIER__ tiled_partition_internal(const thread_block& g) - : thread_block_tile(g) {} -}; - -} // namespace impl - -/** \brief User exposed API to partition groups. - * - * \details This constructs a templated class derieved from thread_group. - * The template defines tile size of the new thread group at compile time. - */ -template -__CG_QUALIFIER__ thread_block_tile tiled_partition(const ParentCGTy& g) { - static_assert(is_valid_tile_size::value, - "Tiled partition with size > wavefront size. Currently not supported "); - return impl::tiled_partition_internal(g); -} -} // namespace cooperative_groups - -#endif // __cplusplus -#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_H diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_fp16.h b/src/hipamd/include/hip/amd_detail/amd_hip_fp16.h deleted file mode 100644 index 2b553bada9..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_fp16.h +++ /dev/null @@ -1,1663 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP16_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP16_H - -#include - -#include "hip/amd_detail/host_defines.h" -#if defined(__HIPCC_RTC__) - #define __HOST_DEVICE__ __device__ -#else - #define __HOST_DEVICE__ __host__ __device__ - #include - #if defined(__cplusplus) - #include - #include - #include -#endif -#endif // !defined(__HIPCC_RTC__) - -#if __HIP_CLANG_ONLY__ - typedef _Float16 _Float16_2 __attribute__((ext_vector_type(2))); - - struct __half_raw { - union { - static_assert(sizeof(_Float16) == sizeof(unsigned short), ""); - - _Float16 data; - unsigned short x; - }; - }; - - struct __half2_raw { - union { - static_assert(sizeof(_Float16_2) == sizeof(unsigned short[2]), ""); - - _Float16_2 data; - struct { - unsigned short x; - unsigned short y; - }; - }; - }; - - #if defined(__cplusplus) - #include "hip_fp16_math_fwd.h" - #include "amd_hip_vector_types.h" - #include "host_defines.h" - - namespace std - { - template<> struct is_floating_point<_Float16> : std::true_type {}; - } - - template - using Enable_if_t = typename std::enable_if::type; - - // BEGIN STRUCT __HALF - struct __half { - protected: - union { - static_assert(sizeof(_Float16) == sizeof(unsigned short), ""); - - _Float16 data; - unsigned short __x; - }; - public: - // CREATORS - __HOST_DEVICE__ - __half() = default; - __HOST_DEVICE__ - __half(const __half_raw& x) : data{x.data} {} - #if !defined(__HIP_NO_HALF_CONVERSIONS__) - __HOST_DEVICE__ - __half(decltype(data) x) : data{x} {} - template< - typename T, - Enable_if_t{}>* = nullptr> - __HOST_DEVICE__ - __half(T x) : data{static_cast<_Float16>(x)} {} - #endif - __HOST_DEVICE__ - __half(const __half&) = default; - __HOST_DEVICE__ - __half(__half&&) = default; - __HOST_DEVICE__ - ~__half() = default; - - // CREATORS - DEVICE ONLY - #if !defined(__HIP_NO_HALF_CONVERSIONS__) - template< - typename T, Enable_if_t{}>* = nullptr> - __HOST_DEVICE__ - __half(T x) : data{static_cast<_Float16>(x)} {} - #endif - - // MANIPULATORS - __HOST_DEVICE__ - __half& operator=(const __half&) = default; - __HOST_DEVICE__ - __half& operator=(__half&&) = default; - __HOST_DEVICE__ - __half& operator=(const __half_raw& x) - { - data = x.data; - return *this; - } - __HOST_DEVICE__ - volatile __half& operator=(const __half_raw& x) volatile - { - data = x.data; - return *this; - } - volatile __half& operator=(const volatile __half_raw& x) volatile - { - data = x.data; - return *this; - } - __half& operator=(__half_raw&& x) - { - data = x.data; - return *this; - } - volatile __half& operator=(__half_raw&& x) volatile - { - data = x.data; - return *this; - } - volatile __half& operator=(volatile __half_raw&& x) volatile - { - data = x.data; - return *this; - } - #if !defined(__HIP_NO_HALF_CONVERSIONS__) - template< - typename T, - Enable_if_t{}>* = nullptr> - __HOST_DEVICE__ - __half& operator=(T x) - { - data = static_cast<_Float16>(x); - return *this; - } - #endif - - // MANIPULATORS - DEVICE ONLY - #if !defined(__HIP_NO_HALF_CONVERSIONS__) - template< - typename T, Enable_if_t{}>* = nullptr> - __device__ - __half& operator=(T x) - { - data = static_cast<_Float16>(x); - return *this; - } - #endif - - #if !defined(__HIP_NO_HALF_OPERATORS__) - __device__ - __half& operator+=(const __half& x) - { - data += x.data; - return *this; - } - __device__ - __half& operator-=(const __half& x) - { - data -= x.data; - return *this; - } - __device__ - __half& operator*=(const __half& x) - { - data *= x.data; - return *this; - } - __device__ - __half& operator/=(const __half& x) - { - data /= x.data; - return *this; - } - __device__ - __half& operator++() { ++data; return *this; } - __device__ - __half operator++(int) - { - __half tmp{*this}; - ++*this; - return tmp; - } - __device__ - __half& operator--() { --data; return *this; } - __device__ - __half operator--(int) - { - __half tmp{*this}; - --*this; - return tmp; - } - #endif - - // ACCESSORS - #if !defined(__HIP_NO_HALF_CONVERSIONS__) - template< - typename T, - Enable_if_t{}>* = nullptr> - __HOST_DEVICE__ - operator T() const { return data; } - #endif - __HOST_DEVICE__ - operator __half_raw() const { return __half_raw{data}; } - __HOST_DEVICE__ - operator __half_raw() const volatile - { - return __half_raw{data}; - } - - #if !defined(__HIP_NO_HALF_CONVERSIONS__) - template< - typename T, Enable_if_t{}>* = nullptr> - __HOST_DEVICE__ - operator T() const { return data; } - #endif - - #if !defined(__HIP_NO_HALF_OPERATORS__) - __device__ - __half operator+() const { return *this; } - __device__ - __half operator-() const - { - __half tmp{*this}; - tmp.data = -tmp.data; - return tmp; - } - #endif - - // FRIENDS - #if !defined(__HIP_NO_HALF_OPERATORS__) - friend - inline - __device__ - __half operator+(const __half& x, const __half& y) - { - return __half{x} += y; - } - friend - inline - __device__ - __half operator-(const __half& x, const __half& y) - { - return __half{x} -= y; - } - friend - inline - __device__ - __half operator*(const __half& x, const __half& y) - { - return __half{x} *= y; - } - friend - inline - __device__ - __half operator/(const __half& x, const __half& y) - { - return __half{x} /= y; - } - friend - inline - __device__ - bool operator==(const __half& x, const __half& y) - { - return x.data == y.data; - } - friend - inline - __device__ - bool operator!=(const __half& x, const __half& y) - { - return !(x == y); - } - friend - inline - __device__ - bool operator<(const __half& x, const __half& y) - { - return x.data < y.data; - } - friend - inline - __device__ - bool operator>(const __half& x, const __half& y) - { - return y.data < x.data; - } - friend - inline - __device__ - bool operator<=(const __half& x, const __half& y) - { - return !(y < x); - } - friend - inline - __device__ - bool operator>=(const __half& x, const __half& y) - { - return !(x < y); - } - #endif // !defined(__HIP_NO_HALF_OPERATORS__) - }; - // END STRUCT __HALF - - // BEGIN STRUCT __HALF2 - struct __half2 { - public: - union { - static_assert( - sizeof(_Float16_2) == sizeof(unsigned short[2]), ""); - - _Float16_2 data; - struct { - unsigned short x; - unsigned short y; - }; - }; - - // CREATORS - __HOST_DEVICE__ - __half2() = default; - __HOST_DEVICE__ - __half2(const __half2_raw& x) : data{x.data} {} - __HOST_DEVICE__ - __half2(decltype(data) x) : data{x} {} - __HOST_DEVICE__ - __half2(const __half& x, const __half& y) - : - data{ - static_cast<__half_raw>(x).data, - static_cast<__half_raw>(y).data} - {} - __HOST_DEVICE__ - __half2(const __half2&) = default; - __HOST_DEVICE__ - __half2(__half2&&) = default; - __HOST_DEVICE__ - ~__half2() = default; - - // MANIPULATORS - __HOST_DEVICE__ - __half2& operator=(const __half2&) = default; - __HOST_DEVICE__ - __half2& operator=(__half2&&) = default; - __HOST_DEVICE__ - __half2& operator=(const __half2_raw& x) - { - data = x.data; - return *this; - } - - // MANIPULATORS - DEVICE ONLY - #if !defined(__HIP_NO_HALF_OPERATORS__) - __device__ - __half2& operator+=(const __half2& x) - { - data += x.data; - return *this; - } - __device__ - __half2& operator-=(const __half2& x) - { - data -= x.data; - return *this; - } - __device__ - __half2& operator*=(const __half2& x) - { - data *= x.data; - return *this; - } - __device__ - __half2& operator/=(const __half2& x) - { - data /= x.data; - return *this; - } - __device__ - __half2& operator++() { return *this += _Float16_2{1, 1}; } - __device__ - __half2 operator++(int) - { - __half2 tmp{*this}; - ++*this; - return tmp; - } - __device__ - __half2& operator--() { return *this -= _Float16_2{1, 1}; } - __device__ - __half2 operator--(int) - { - __half2 tmp{*this}; - --*this; - return tmp; - } - #endif - - // ACCESSORS - __HOST_DEVICE__ - operator decltype(data)() const { return data; } - __HOST_DEVICE__ - operator __half2_raw() const { return __half2_raw{data}; } - - // ACCESSORS - DEVICE ONLY - #if !defined(__HIP_NO_HALF_OPERATORS__) - __device__ - __half2 operator+() const { return *this; } - __device__ - __half2 operator-() const - { - __half2 tmp{*this}; - tmp.data = -tmp.data; - return tmp; - } - #endif - - // FRIENDS - #if !defined(__HIP_NO_HALF_OPERATORS__) - friend - inline - __device__ - __half2 operator+(const __half2& x, const __half2& y) - { - return __half2{x} += y; - } - friend - inline - __device__ - __half2 operator-(const __half2& x, const __half2& y) - { - return __half2{x} -= y; - } - friend - inline - __device__ - __half2 operator*(const __half2& x, const __half2& y) - { - return __half2{x} *= y; - } - friend - inline - __device__ - __half2 operator/(const __half2& x, const __half2& y) - { - return __half2{x} /= y; - } - friend - inline - __device__ - bool operator==(const __half2& x, const __half2& y) - { - auto r = x.data == y.data; - return r.x != 0 && r.y != 0; - } - friend - inline - __device__ - bool operator!=(const __half2& x, const __half2& y) - { - return !(x == y); - } - friend - inline - __device__ - bool operator<(const __half2& x, const __half2& y) - { - auto r = x.data < y.data; - return r.x != 0 && r.y != 0; - } - friend - inline - __device__ - bool operator>(const __half2& x, const __half2& y) - { - return y < x; - } - friend - inline - __device__ - bool operator<=(const __half2& x, const __half2& y) - { - return !(y < x); - } - friend - inline - __device__ - bool operator>=(const __half2& x, const __half2& y) - { - return !(x < y); - } - #endif // !defined(__HIP_NO_HALF_OPERATORS__) - }; - // END STRUCT __HALF2 - - namespace - { - inline - __HOST_DEVICE__ - __half2 make_half2(__half x, __half y) - { - return __half2{x, y}; - } - - inline - __HOST_DEVICE__ - __half __low2half(__half2 x) - { - return __half{__half_raw{static_cast<__half2_raw>(x).data.x}}; - } - - inline - __HOST_DEVICE__ - __half __high2half(__half2 x) - { - return __half{__half_raw{static_cast<__half2_raw>(x).data.y}}; - } - - inline - __HOST_DEVICE__ - __half2 __half2half2(__half x) - { - return __half2{x, x}; - } - - inline - __HOST_DEVICE__ - __half2 __halves2half2(__half x, __half y) - { - return __half2{x, y}; - } - - inline - __HOST_DEVICE__ - __half2 __low2half2(__half2 x) - { - return __half2{ - _Float16_2{ - static_cast<__half2_raw>(x).data.x, - static_cast<__half2_raw>(x).data.x}}; - } - - inline - __HOST_DEVICE__ - __half2 __high2half2(__half2 x) - { - return __half2_raw{ - _Float16_2{ - static_cast<__half2_raw>(x).data.y, - static_cast<__half2_raw>(x).data.y}}; - } - - inline - __HOST_DEVICE__ - __half2 __lows2half2(__half2 x, __half2 y) - { - return __half2_raw{ - _Float16_2{ - static_cast<__half2_raw>(x).data.x, - static_cast<__half2_raw>(y).data.x}}; - } - - inline - __HOST_DEVICE__ - __half2 __highs2half2(__half2 x, __half2 y) - { - return __half2_raw{ - _Float16_2{ - static_cast<__half2_raw>(x).data.y, - static_cast<__half2_raw>(y).data.y}}; - } - - inline - __HOST_DEVICE__ - __half2 __lowhigh2highlow(__half2 x) - { - return __half2_raw{ - _Float16_2{ - static_cast<__half2_raw>(x).data.y, - static_cast<__half2_raw>(x).data.x}}; - } - - // Bitcasts - inline - __device__ - short __half_as_short(__half x) - { - return static_cast<__half_raw>(x).x; - } - - inline - __device__ - unsigned short __half_as_ushort(__half x) - { - return static_cast<__half_raw>(x).x; - } - - inline - __device__ - __half __short_as_half(short x) - { - __half_raw r; r.x = x; - return r; - } - - inline - __device__ - __half __ushort_as_half(unsigned short x) - { - __half_raw r; r.x = x; - return r; - } - - // TODO: rounding behaviour is not correct. - // float -> half | half2 - inline - __HOST_DEVICE__ - __half __float2half(float x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __HOST_DEVICE__ - __half __float2half_rn(float x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __HOST_DEVICE__ - __half __float2half_rz(float x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __HOST_DEVICE__ - __half __float2half_rd(float x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __HOST_DEVICE__ - __half __float2half_ru(float x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __HOST_DEVICE__ - __half2 __float2half2_rn(float x) - { - return __half2_raw{ - _Float16_2{ - static_cast<_Float16>(x), static_cast<_Float16>(x)}}; - } - inline - __HOST_DEVICE__ - __half2 __floats2half2_rn(float x, float y) - { - return __half2_raw{_Float16_2{ - static_cast<_Float16>(x), static_cast<_Float16>(y)}}; - } - inline - __HOST_DEVICE__ - __half2 __float22half2_rn(float2 x) - { - return __floats2half2_rn(x.x, x.y); - } - - // half | half2 -> float - inline - __HOST_DEVICE__ - float __half2float(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __HOST_DEVICE__ - float __low2float(__half2 x) - { - return static_cast<__half2_raw>(x).data.x; - } - inline - __HOST_DEVICE__ - float __high2float(__half2 x) - { - return static_cast<__half2_raw>(x).data.y; - } - inline - __HOST_DEVICE__ - float2 __half22float2(__half2 x) - { - return make_float2( - static_cast<__half2_raw>(x).data.x, - static_cast<__half2_raw>(x).data.y); - } - - // half -> int - inline - __device__ - int __half2int_rn(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - int __half2int_rz(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - int __half2int_rd(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - int __half2int_ru(__half x) - { - return static_cast<__half_raw>(x).data; - } - - // int -> half - inline - __device__ - __half __int2half_rn(int x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __int2half_rz(int x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __int2half_rd(int x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __int2half_ru(int x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - - // half -> short - inline - __device__ - short __half2short_rn(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - short __half2short_rz(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - short __half2short_rd(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - short __half2short_ru(__half x) - { - return static_cast<__half_raw>(x).data; - } - - // short -> half - inline - __device__ - __half __short2half_rn(short x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __short2half_rz(short x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __short2half_rd(short x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __short2half_ru(short x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - - // half -> long long - inline - __device__ - long long __half2ll_rn(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - long long __half2ll_rz(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - long long __half2ll_rd(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - long long __half2ll_ru(__half x) - { - return static_cast<__half_raw>(x).data; - } - - // long long -> half - inline - __device__ - __half __ll2half_rn(long long x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ll2half_rz(long long x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ll2half_rd(long long x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ll2half_ru(long long x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - - // half -> unsigned int - inline - __device__ - unsigned int __half2uint_rn(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned int __half2uint_rz(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned int __half2uint_rd(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned int __half2uint_ru(__half x) - { - return static_cast<__half_raw>(x).data; - } - - // unsigned int -> half - inline - __device__ - __half __uint2half_rn(unsigned int x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __uint2half_rz(unsigned int x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __uint2half_rd(unsigned int x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __uint2half_ru(unsigned int x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - - // half -> unsigned short - inline - __device__ - unsigned short __half2ushort_rn(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned short __half2ushort_rz(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned short __half2ushort_rd(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned short __half2ushort_ru(__half x) - { - return static_cast<__half_raw>(x).data; - } - - // unsigned short -> half - inline - __device__ - __half __ushort2half_rn(unsigned short x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ushort2half_rz(unsigned short x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ushort2half_rd(unsigned short x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ushort2half_ru(unsigned short x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - - // half -> unsigned long long - inline - __device__ - unsigned long long __half2ull_rn(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned long long __half2ull_rz(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned long long __half2ull_rd(__half x) - { - return static_cast<__half_raw>(x).data; - } - inline - __device__ - unsigned long long __half2ull_ru(__half x) - { - return static_cast<__half_raw>(x).data; - } - - // unsigned long long -> half - inline - __device__ - __half __ull2half_rn(unsigned long long x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ull2half_rz(unsigned long long x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ull2half_rd(unsigned long long x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - inline - __device__ - __half __ull2half_ru(unsigned long long x) - { - return __half_raw{static_cast<_Float16>(x)}; - } - - // Load primitives - inline - __device__ - __half __ldg(const __half* ptr) { return *ptr; } - inline - __device__ - __half __ldcg(const __half* ptr) { return *ptr; } - inline - __device__ - __half __ldca(const __half* ptr) { return *ptr; } - inline - __device__ - __half __ldcs(const __half* ptr) { return *ptr; } - - inline - __HOST_DEVICE__ - __half2 __ldg(const __half2* ptr) { return *ptr; } - inline - __HOST_DEVICE__ - __half2 __ldcg(const __half2* ptr) { return *ptr; } - inline - __HOST_DEVICE__ - __half2 __ldca(const __half2* ptr) { return *ptr; } - inline - __HOST_DEVICE__ - __half2 __ldcs(const __half2* ptr) { return *ptr; } - - // Relations - inline - __device__ - bool __heq(__half x, __half y) - { - return static_cast<__half_raw>(x).data == - static_cast<__half_raw>(y).data; - } - inline - __device__ - bool __hne(__half x, __half y) - { - return static_cast<__half_raw>(x).data != - static_cast<__half_raw>(y).data; - } - inline - __device__ - bool __hle(__half x, __half y) - { - return static_cast<__half_raw>(x).data <= - static_cast<__half_raw>(y).data; - } - inline - __device__ - bool __hge(__half x, __half y) - { - return static_cast<__half_raw>(x).data >= - static_cast<__half_raw>(y).data; - } - inline - __device__ - bool __hlt(__half x, __half y) - { - return static_cast<__half_raw>(x).data < - static_cast<__half_raw>(y).data; - } - inline - __device__ - bool __hgt(__half x, __half y) - { - return static_cast<__half_raw>(x).data > - static_cast<__half_raw>(y).data; - } - inline - __device__ - bool __hequ(__half x, __half y) { return __heq(x, y); } - inline - __device__ - bool __hneu(__half x, __half y) { return __hne(x, y); } - inline - __device__ - bool __hleu(__half x, __half y) { return __hle(x, y); } - inline - __device__ - bool __hgeu(__half x, __half y) { return __hge(x, y); } - inline - __device__ - bool __hltu(__half x, __half y) { return __hlt(x, y); } - inline - __device__ - bool __hgtu(__half x, __half y) { return __hgt(x, y); } - - inline - __HOST_DEVICE__ - __half2 __heq2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(x).data == - static_cast<__half2_raw>(y).data; - return __builtin_convertvector(-r, _Float16_2); - } - inline - __HOST_DEVICE__ - __half2 __hne2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(x).data != - static_cast<__half2_raw>(y).data; - return __builtin_convertvector(-r, _Float16_2); - } - inline - __HOST_DEVICE__ - __half2 __hle2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(x).data <= - static_cast<__half2_raw>(y).data; - return __builtin_convertvector(-r, _Float16_2); - } - inline - __HOST_DEVICE__ - __half2 __hge2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(x).data >= - static_cast<__half2_raw>(y).data; - return __builtin_convertvector(-r, _Float16_2); - } - inline - __HOST_DEVICE__ - __half2 __hlt2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(x).data < - static_cast<__half2_raw>(y).data; - return __builtin_convertvector(-r, _Float16_2); - } - inline - __HOST_DEVICE__ - __half2 __hgt2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(x).data > - static_cast<__half2_raw>(y).data; - return __builtin_convertvector(-r, _Float16_2); - } - inline - __HOST_DEVICE__ - __half2 __hequ2(__half2 x, __half2 y) { return __heq2(x, y); } - inline - __HOST_DEVICE__ - __half2 __hneu2(__half2 x, __half2 y) { return __hne2(x, y); } - inline - __HOST_DEVICE__ - __half2 __hleu2(__half2 x, __half2 y) { return __hle2(x, y); } - inline - __HOST_DEVICE__ - __half2 __hgeu2(__half2 x, __half2 y) { return __hge2(x, y); } - inline - __HOST_DEVICE__ - __half2 __hltu2(__half2 x, __half2 y) { return __hlt2(x, y); } - inline - __HOST_DEVICE__ - __half2 __hgtu2(__half2 x, __half2 y) { return __hgt2(x, y); } - - inline - __HOST_DEVICE__ - bool __hbeq2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__heq2(x, y)); - return r.data.x != 0 && r.data.y != 0; - } - inline - __HOST_DEVICE__ - bool __hbne2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__hne2(x, y)); - return r.data.x != 0 && r.data.y != 0; - } - inline - __HOST_DEVICE__ - bool __hble2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__hle2(x, y)); - return r.data.x != 0 && r.data.y != 0; - } - inline - __HOST_DEVICE__ - bool __hbge2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__hge2(x, y)); - return r.data.x != 0 && r.data.y != 0; - } - inline - __HOST_DEVICE__ - bool __hblt2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__hlt2(x, y)); - return r.data.x != 0 && r.data.y != 0; - } - inline - __HOST_DEVICE__ - bool __hbgt2(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__hgt2(x, y)); - return r.data.x != 0 && r.data.y != 0; - } - inline - __HOST_DEVICE__ - bool __hbequ2(__half2 x, __half2 y) { return __hbeq2(x, y); } - inline - __HOST_DEVICE__ - bool __hbneu2(__half2 x, __half2 y) { return __hbne2(x, y); } - inline - __HOST_DEVICE__ - bool __hbleu2(__half2 x, __half2 y) { return __hble2(x, y); } - inline - __HOST_DEVICE__ - bool __hbgeu2(__half2 x, __half2 y) { return __hbge2(x, y); } - inline - __HOST_DEVICE__ - bool __hbltu2(__half2 x, __half2 y) { return __hblt2(x, y); } - inline - __HOST_DEVICE__ - bool __hbgtu2(__half2 x, __half2 y) { return __hbgt2(x, y); } - - // Arithmetic - inline - __device__ - __half __clamp_01(__half x) - { - auto r = static_cast<__half_raw>(x); - - if (__hlt(x, __half_raw{0})) return __half_raw{0}; - if (__hlt(__half_raw{1}, x)) return __half_raw{1}; - return r; - } - - inline - __device__ - __half __hadd(__half x, __half y) - { - return __half_raw{ - static_cast<__half_raw>(x).data + - static_cast<__half_raw>(y).data}; - } - inline - __device__ - __half __habs(__half x) - { - return __half_raw{ - __ocml_fabs_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half __hsub(__half x, __half y) - { - return __half_raw{ - static_cast<__half_raw>(x).data - - static_cast<__half_raw>(y).data}; - } - inline - __device__ - __half __hmul(__half x, __half y) - { - return __half_raw{ - static_cast<__half_raw>(x).data * - static_cast<__half_raw>(y).data}; - } - inline - __device__ - __half __hadd_sat(__half x, __half y) - { - return __clamp_01(__hadd(x, y)); - } - inline - __device__ - __half __hsub_sat(__half x, __half y) - { - return __clamp_01(__hsub(x, y)); - } - inline - __device__ - __half __hmul_sat(__half x, __half y) - { - return __clamp_01(__hmul(x, y)); - } - inline - __device__ - __half __hfma(__half x, __half y, __half z) - { - return __half_raw{__ocml_fma_f16( - static_cast<__half_raw>(x).data, - static_cast<__half_raw>(y).data, - static_cast<__half_raw>(z).data)}; - } - inline - __device__ - __half __hfma_sat(__half x, __half y, __half z) - { - return __clamp_01(__hfma(x, y, z)); - } - inline - __device__ - __half __hdiv(__half x, __half y) - { - return __half_raw{ - static_cast<__half_raw>(x).data / - static_cast<__half_raw>(y).data}; - } - - inline - __HOST_DEVICE__ - __half2 __hadd2(__half2 x, __half2 y) - { - return __half2_raw{ - static_cast<__half2_raw>(x).data + - static_cast<__half2_raw>(y).data}; - } - inline - __HOST_DEVICE__ - __half2 __habs2(__half2 x) - { - return __half2_raw{ - __ocml_fabs_2f16(static_cast<__half2_raw>(x).data)}; - } - inline - __HOST_DEVICE__ - __half2 __hsub2(__half2 x, __half2 y) - { - return __half2_raw{ - static_cast<__half2_raw>(x).data - - static_cast<__half2_raw>(y).data}; - } - inline - __HOST_DEVICE__ - __half2 __hmul2(__half2 x, __half2 y) - { - return __half2_raw{ - static_cast<__half2_raw>(x).data * - static_cast<__half2_raw>(y).data}; - } - inline - __HOST_DEVICE__ - __half2 __hadd2_sat(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__hadd2(x, y)); - return __half2{ - __clamp_01(__half_raw{r.data.x}), - __clamp_01(__half_raw{r.data.y})}; - } - inline - __HOST_DEVICE__ - __half2 __hsub2_sat(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__hsub2(x, y)); - return __half2{ - __clamp_01(__half_raw{r.data.x}), - __clamp_01(__half_raw{r.data.y})}; - } - inline - __HOST_DEVICE__ - __half2 __hmul2_sat(__half2 x, __half2 y) - { - auto r = static_cast<__half2_raw>(__hmul2(x, y)); - return __half2{ - __clamp_01(__half_raw{r.data.x}), - __clamp_01(__half_raw{r.data.y})}; - } - inline - __HOST_DEVICE__ - __half2 __hfma2(__half2 x, __half2 y, __half2 z) - { - return __half2_raw{__ocml_fma_2f16(x, y, z)}; - } - inline - __HOST_DEVICE__ - __half2 __hfma2_sat(__half2 x, __half2 y, __half2 z) - { - auto r = static_cast<__half2_raw>(__hfma2(x, y, z)); - return __half2{ - __clamp_01(__half_raw{r.data.x}), - __clamp_01(__half_raw{r.data.y})}; - } - inline - __HOST_DEVICE__ - __half2 __h2div(__half2 x, __half2 y) - { - return __half2_raw{ - static_cast<__half2_raw>(x).data / - static_cast<__half2_raw>(y).data}; - } - - // Math functions - #if __HIP_CLANG_ONLY__ - inline - __device__ - float amd_mixed_dot(__half2 a, __half2 b, float c, bool saturate) { - return __ockl_fdot2(static_cast<__half2_raw>(a).data, - static_cast<__half2_raw>(b).data, - c, saturate); - } - #endif - inline - __device__ - __half htrunc(__half x) - { - return __half_raw{ - __ocml_trunc_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hceil(__half x) - { - return __half_raw{ - __ocml_ceil_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hfloor(__half x) - { - return __half_raw{ - __ocml_floor_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hrint(__half x) - { - return __half_raw{ - __ocml_rint_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hsin(__half x) - { - return __half_raw{ - __ocml_sin_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hcos(__half x) - { - return __half_raw{ - __ocml_cos_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hexp(__half x) - { - return __half_raw{ - __ocml_exp_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hexp2(__half x) - { - return __half_raw{ - __ocml_exp2_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hexp10(__half x) - { - return __half_raw{ - __ocml_exp10_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hlog2(__half x) - { - return __half_raw{ - __ocml_log2_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hlog(__half x) - { - return __half_raw{ - __ocml_log_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hlog10(__half x) - { - return __half_raw{ - __ocml_log10_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hrcp(__half x) - { - return __half_raw{ - __llvm_amdgcn_rcp_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hrsqrt(__half x) - { - return __half_raw{ - __ocml_rsqrt_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - __half hsqrt(__half x) - { - return __half_raw{ - __ocml_sqrt_f16(static_cast<__half_raw>(x).data)}; - } - inline - __device__ - bool __hisinf(__half x) - { - return __ocml_isinf_f16(static_cast<__half_raw>(x).data); - } - inline - __device__ - bool __hisnan(__half x) - { - return __ocml_isnan_f16(static_cast<__half_raw>(x).data); - } - inline - __device__ - __half __hneg(__half x) - { - return __half_raw{-static_cast<__half_raw>(x).data}; - } - - inline - __HOST_DEVICE__ - __half2 h2trunc(__half2 x) - { - return __half2_raw{__ocml_trunc_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2ceil(__half2 x) - { - return __half2_raw{__ocml_ceil_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2floor(__half2 x) - { - return __half2_raw{__ocml_floor_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2rint(__half2 x) - { - return __half2_raw{__ocml_rint_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2sin(__half2 x) - { - return __half2_raw{__ocml_sin_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2cos(__half2 x) - { - return __half2_raw{__ocml_cos_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2exp(__half2 x) - { - return __half2_raw{__ocml_exp_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2exp2(__half2 x) - { - return __half2_raw{__ocml_exp2_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2exp10(__half2 x) - { - return __half2_raw{__ocml_exp10_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2log2(__half2 x) - { - return __half2_raw{__ocml_log2_2f16(x)}; - } - inline - __HOST_DEVICE__ - __half2 h2log(__half2 x) { return __ocml_log_2f16(x); } - inline - __HOST_DEVICE__ - __half2 h2log10(__half2 x) { return __ocml_log10_2f16(x); } - inline - __HOST_DEVICE__ - __half2 h2rcp(__half2 x) { return __llvm_amdgcn_rcp_2f16(x); } - inline - __HOST_DEVICE__ - __half2 h2rsqrt(__half2 x) { return __ocml_rsqrt_2f16(x); } - inline - __HOST_DEVICE__ - __half2 h2sqrt(__half2 x) { return __ocml_sqrt_2f16(x); } - inline - __HOST_DEVICE__ - __half2 __hisinf2(__half2 x) - { - auto r = __ocml_isinf_2f16(x); - return __half2_raw{_Float16_2{ - static_cast<_Float16>(r.x), static_cast<_Float16>(r.y)}}; - } - inline - __HOST_DEVICE__ - __half2 __hisnan2(__half2 x) - { - auto r = __ocml_isnan_2f16(x); - return __half2_raw{_Float16_2{ - static_cast<_Float16>(r.x), static_cast<_Float16>(r.y)}}; - } - inline - __HOST_DEVICE__ - __half2 __hneg2(__half2 x) - { - return __half2_raw{-static_cast<__half2_raw>(x).data}; - } - } // Anonymous namespace. - - #if !defined(HIP_NO_HALF) - using half = __half; - using half2 = __half2; - #endif - #endif // defined(__cplusplus) -#elif defined(__GNUC__) - #include "hip_fp16_gcc.h" -#endif // !defined(__clang__) && defined(__GNUC__) - -#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP16_H diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_runtime.h b/src/hipamd/include/hip/amd_detail/amd_hip_runtime.h deleted file mode 100644 index 15952722b8..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_runtime.h +++ /dev/null @@ -1,417 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/hip_runtime.h - * @brief Contains definitions of APIs for HIP runtime. - */ - -//#pragma once -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H - -#include - -//--- -// Top part of file can be compiled with any compiler - -#if !defined(__HIPCC_RTC__) -//#include -#if __cplusplus -#include -#include -#else -#include -#include -#include -#endif // __cplusplus -#endif // !defined(__HIPCC_RTC__) - -// __hip_malloc is not working. Disable it by default. -#ifndef __HIP_ENABLE_DEVICE_MALLOC__ -#define __HIP_ENABLE_DEVICE_MALLOC__ 0 -#endif - -#if __HIP_CLANG_ONLY__ - -#if !defined(__align__) -#define __align__(x) __attribute__((aligned(x))) -#endif - -#define CUDA_SUCCESS hipSuccess - -#if !defined(__HIPCC_RTC__) -#include -extern int HIP_TRACE_API; -#endif // !defined(__HIPCC_RTC__) - -#ifdef __cplusplus -#include -#endif -#include -#include -#include -#include -#include -#include - -// TODO-HCC remove old definitions ; ~1602 hcc supports __HCC_ACCELERATOR__ define. -#if defined(__KALMAR_ACCELERATOR__) && !defined(__HCC_ACCELERATOR__) -#define __HCC_ACCELERATOR__ __KALMAR_ACCELERATOR__ -#endif - -// Feature tests: -#if (defined(__HCC_ACCELERATOR__) && (__HCC_ACCELERATOR__ != 0)) || __HIP_DEVICE_COMPILE__ -// Device compile and not host compile: - -// 32-bit Atomics: -#define __HIP_ARCH_HAS_GLOBAL_INT32_ATOMICS__ (1) -#define __HIP_ARCH_HAS_GLOBAL_FLOAT_ATOMIC_EXCH__ (1) -#define __HIP_ARCH_HAS_SHARED_INT32_ATOMICS__ (1) -#define __HIP_ARCH_HAS_SHARED_FLOAT_ATOMIC_EXCH__ (1) -#define __HIP_ARCH_HAS_FLOAT_ATOMIC_ADD__ (1) - -// 64-bit Atomics: -#define __HIP_ARCH_HAS_GLOBAL_INT64_ATOMICS__ (1) -#define __HIP_ARCH_HAS_SHARED_INT64_ATOMICS__ (1) - -// Doubles -#define __HIP_ARCH_HAS_DOUBLES__ (1) - -// warp cross-lane operations: -#define __HIP_ARCH_HAS_WARP_VOTE__ (1) -#define __HIP_ARCH_HAS_WARP_BALLOT__ (1) -#define __HIP_ARCH_HAS_WARP_SHUFFLE__ (1) -#define __HIP_ARCH_HAS_WARP_FUNNEL_SHIFT__ (0) - -// sync -#define __HIP_ARCH_HAS_THREAD_FENCE_SYSTEM__ (1) -#define __HIP_ARCH_HAS_SYNC_THREAD_EXT__ (0) - -// misc -#define __HIP_ARCH_HAS_SURFACE_FUNCS__ (0) -#define __HIP_ARCH_HAS_3DGRID__ (1) -#define __HIP_ARCH_HAS_DYNAMIC_PARALLEL__ (0) - -#endif /* Device feature flags */ - - -#define launch_bounds_impl0(requiredMaxThreadsPerBlock) \ - __attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock))) -#define launch_bounds_impl1(requiredMaxThreadsPerBlock, minBlocksPerMultiprocessor) \ - __attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock), \ - amdgpu_waves_per_eu(minBlocksPerMultiprocessor))) -#define select_impl_(_1, _2, impl_, ...) impl_ -#define __launch_bounds__(...) \ - select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__) - -#if !defined(__HIPCC_RTC__) -__host__ inline void* __get_dynamicgroupbaseptr() { return nullptr; } -#endif // !defined(__HIPCC_RTC__) - -#if __HIP_ARCH_GFX701__ == 0 - -__device__ unsigned __hip_ds_bpermute(int index, unsigned src); -__device__ float __hip_ds_bpermutef(int index, float src); -__device__ unsigned __hip_ds_permute(int index, unsigned src); -__device__ float __hip_ds_permutef(int index, float src); - -template -__device__ unsigned __hip_ds_swizzle_N(unsigned int src); -template -__device__ float __hip_ds_swizzlef_N(float src); - -template -__device__ int __hip_move_dpp_N(int src); - -#endif //__HIP_ARCH_GFX803__ == 1 - -#ifndef __OPENMP_AMDGCN__ -#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -#if __HIP_ENABLE_DEVICE_MALLOC__ -extern "C" __device__ void* __hip_malloc(size_t); -extern "C" __device__ void* __hip_free(void* ptr); -static inline __device__ void* malloc(size_t size) { return __hip_malloc(size); } -static inline __device__ void* free(void* ptr) { return __hip_free(ptr); } -#else -static inline __device__ void* malloc(size_t size) { __builtin_trap(); return nullptr; } -static inline __device__ void* free(void* ptr) { __builtin_trap(); return nullptr; } -#endif -#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -#endif // !__OPENMP_AMDGCN__ - -// End doxygen API: -/** - * @} - */ - -// -// hip-clang functions -// -#if !defined(__HIPCC_RTC__) -#define HIP_KERNEL_NAME(...) __VA_ARGS__ -#define HIP_SYMBOL(X) X - -typedef int hipLaunchParm; - -template ::type* = nullptr> -void pArgs(const std::tuple&, void*) {} - -template ::type* = nullptr> -void pArgs(const std::tuple& formals, void** _vargs) { - using T = typename std::tuple_element >::type; - - static_assert(!std::is_reference{}, - "A __global__ function cannot have a reference as one of its " - "arguments."); -#if defined(HIP_STRICT) - static_assert(std::is_trivially_copyable{}, - "Only TriviallyCopyable types can be arguments to a __global__ " - "function"); -#endif - _vargs[n] = const_cast(reinterpret_cast(&std::get(formals))); - return pArgs(formals, _vargs); -} - -template -std::tuple validateArgsCountType(void (*kernel)(Formals...), std::tuple(actuals)) { - static_assert(sizeof...(Formals) == sizeof...(Actuals), "Argument Count Mismatch"); - std::tuple to_formals{std::move(actuals)}; - return to_formals; -} - -#if defined(HIP_TEMPLATE_KERNEL_LAUNCH) -template -void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks, - std::uint32_t sharedMemBytes, hipStream_t stream, Args... args) { - constexpr size_t count = sizeof...(Args); - auto tup_ = std::tuple{args...}; - auto tup = validateArgsCountType(kernel, tup_); - void* _Args[count]; - pArgs<0>(tup, _Args); - - auto k = reinterpret_cast(kernel); - hipLaunchKernel(k, numBlocks, dimBlocks, _Args, sharedMemBytes, stream); -} -#else -#define hipLaunchKernelGGLInternal(kernelName, numBlocks, numThreads, memPerBlock, streamId, ...) \ - do { \ - kernelName<<<(numBlocks), (numThreads), (memPerBlock), (streamId)>>>(__VA_ARGS__); \ - } while (0) - -#define hipLaunchKernelGGL(kernelName, ...) hipLaunchKernelGGLInternal((kernelName), __VA_ARGS__) -#endif - -#include -#endif // !defined(__HIPCC_RTC__) - -extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_id(uint); -extern "C" __device__ __attribute__((const)) size_t __ockl_get_group_id(uint); -extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_size(uint); -extern "C" __device__ __attribute__((const)) size_t __ockl_get_num_groups(uint); -struct __HIP_BlockIdx { - __device__ - std::uint32_t operator()(std::uint32_t x) const noexcept { return __ockl_get_group_id(x); } -}; -struct __HIP_BlockDim { - __device__ - std::uint32_t operator()(std::uint32_t x) const noexcept { - return __ockl_get_local_size(x); - } -}; -struct __HIP_GridDim { - __device__ - std::uint32_t operator()(std::uint32_t x) const noexcept { - return __ockl_get_num_groups(x); - } -}; -struct __HIP_ThreadIdx { - __device__ - std::uint32_t operator()(std::uint32_t x) const noexcept { - return __ockl_get_local_id(x); - } -}; - -#if defined(__HIPCC_RTC__) -typedef struct dim3 { - uint32_t x; ///< x - uint32_t y; ///< y - uint32_t z; ///< z -#ifdef __cplusplus - constexpr __device__ dim3(uint32_t _x = 1, uint32_t _y = 1, uint32_t _z = 1) : x(_x), y(_y), z(_z){}; -#endif -} dim3; -#endif // !defined(__HIPCC_RTC__) - -template -struct __HIP_Coordinates { - using R = decltype(F{}(0)); - - struct __X { __device__ operator R() const noexcept { return F{}(0); } }; - struct __Y { __device__ operator R() const noexcept { return F{}(1); } }; - struct __Z { __device__ operator R() const noexcept { return F{}(2); } }; - - static constexpr __X x{}; - static constexpr __Y y{}; - static constexpr __Z z{}; -#ifdef __cplusplus - __device__ operator dim3() const { return dim3(x, y, z); } -#endif - -}; -template -#if !defined(_MSC_VER) -__attribute__((weak)) -#endif -constexpr typename __HIP_Coordinates::__X __HIP_Coordinates::x; -template -#if !defined(_MSC_VER) -__attribute__((weak)) -#endif -constexpr typename __HIP_Coordinates::__Y __HIP_Coordinates::y; -template -#if !defined(_MSC_VER) -__attribute__((weak)) -#endif -constexpr typename __HIP_Coordinates::__Z __HIP_Coordinates::z; - -extern "C" __device__ __attribute__((const)) size_t __ockl_get_global_size(uint); -inline -__device__ -std::uint32_t operator*(__HIP_Coordinates<__HIP_GridDim>::__X, - __HIP_Coordinates<__HIP_BlockDim>::__X) noexcept { - return __ockl_get_global_size(0); -} -inline -__device__ -std::uint32_t operator*(__HIP_Coordinates<__HIP_BlockDim>::__X, - __HIP_Coordinates<__HIP_GridDim>::__X) noexcept { - return __ockl_get_global_size(0); -} -inline -__device__ -std::uint32_t operator*(__HIP_Coordinates<__HIP_GridDim>::__Y, - __HIP_Coordinates<__HIP_BlockDim>::__Y) noexcept { - return __ockl_get_global_size(1); -} -inline -__device__ -std::uint32_t operator*(__HIP_Coordinates<__HIP_BlockDim>::__Y, - __HIP_Coordinates<__HIP_GridDim>::__Y) noexcept { - return __ockl_get_global_size(1); -} -inline -__device__ -std::uint32_t operator*(__HIP_Coordinates<__HIP_GridDim>::__Z, - __HIP_Coordinates<__HIP_BlockDim>::__Z) noexcept { - return __ockl_get_global_size(2); -} -inline -__device__ -std::uint32_t operator*(__HIP_Coordinates<__HIP_BlockDim>::__Z, - __HIP_Coordinates<__HIP_GridDim>::__Z) noexcept { - return __ockl_get_global_size(2); -} - -static constexpr __HIP_Coordinates<__HIP_BlockDim> blockDim{}; -static constexpr __HIP_Coordinates<__HIP_BlockIdx> blockIdx{}; -static constexpr __HIP_Coordinates<__HIP_GridDim> gridDim{}; -static constexpr __HIP_Coordinates<__HIP_ThreadIdx> threadIdx{}; - -extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_id(uint); -#define hipThreadIdx_x (__ockl_get_local_id(0)) -#define hipThreadIdx_y (__ockl_get_local_id(1)) -#define hipThreadIdx_z (__ockl_get_local_id(2)) - -extern "C" __device__ __attribute__((const)) size_t __ockl_get_group_id(uint); -#define hipBlockIdx_x (__ockl_get_group_id(0)) -#define hipBlockIdx_y (__ockl_get_group_id(1)) -#define hipBlockIdx_z (__ockl_get_group_id(2)) - -extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_size(uint); -#define hipBlockDim_x (__ockl_get_local_size(0)) -#define hipBlockDim_y (__ockl_get_local_size(1)) -#define hipBlockDim_z (__ockl_get_local_size(2)) - -extern "C" __device__ __attribute__((const)) size_t __ockl_get_num_groups(uint); -#define hipGridDim_x (__ockl_get_num_groups(0)) -#define hipGridDim_y (__ockl_get_num_groups(1)) -#define hipGridDim_z (__ockl_get_num_groups(2)) - -#include - -#if __HIP_HCC_COMPAT_MODE__ -// Define HCC work item functions in terms of HIP builtin variables. -#pragma push_macro("__DEFINE_HCC_FUNC") -#define __DEFINE_HCC_FUNC(hc_fun,hip_var) \ -inline __device__ __attribute__((always_inline)) uint hc_get_##hc_fun(uint i) { \ - if (i==0) \ - return hip_var.x; \ - else if(i==1) \ - return hip_var.y; \ - else \ - return hip_var.z; \ -} - -__DEFINE_HCC_FUNC(workitem_id, threadIdx) -__DEFINE_HCC_FUNC(group_id, blockIdx) -__DEFINE_HCC_FUNC(group_size, blockDim) -__DEFINE_HCC_FUNC(num_groups, gridDim) -#pragma pop_macro("__DEFINE_HCC_FUNC") - -extern "C" __device__ __attribute__((const)) size_t __ockl_get_global_id(uint); -inline __device__ __attribute__((always_inline)) uint -hc_get_workitem_absolute_id(int dim) -{ - return (uint)__ockl_get_global_id(dim); -} - -#endif - -#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -#if !defined(__HIPCC_RTC__) -// Support std::complex. -#if !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__ -#pragma push_macro("__CUDA__") -#define __CUDA__ -#include <__clang_cuda_math_forward_declares.h> -#include <__clang_cuda_complex_builtins.h> -// Workaround for using libc++ with HIP-Clang. -// The following headers requires clang include path before standard C++ include path. -// However libc++ include path requires to be before clang include path. -// To workaround this, we pass -isystem with the parent directory of clang include -// path instead of the clang include path itself. -#include -#include -#include -#undef __CUDA__ -#pragma pop_macro("__CUDA__") -#endif // !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__ -#endif // !defined(__HIPCC_RTC__) -#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -#endif // __HIP_CLANG_ONLY__ - -#include - -#endif // HIP_AMD_DETAIL_RUNTIME_H diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_surface_types.h b/src/hipamd/include/hip/amd_detail/amd_hip_surface_types.h deleted file mode 100644 index 27bb84ad39..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_surface_types.h +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright (c) 2015- present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/hip_surface_types.h - * @brief Defines surface types for HIP runtime. - */ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_SURFACE_TYPES_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_SURFACE_TYPES_H - -#include - -/** - * An opaque value that represents a hip surface object - */ -typedef unsigned long long hipSurfaceObject_t; - -/** - * hip surface reference - */ -struct surfaceReference { - hipSurfaceObject_t surfaceObject; -}; - -/** - * hip surface boundary modes - */ -enum hipSurfaceBoundaryMode { - hipBoundaryModeZero = 0, - hipBoundaryModeTrap = 1, - hipBoundaryModeClamp = 2 -}; - -#endif /* !HIP_INCLUDE_HIP_AMD_DETAIL_HIP_SURFACE_TYPES_H */ diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_texture_types.h b/src/hipamd/include/hip/amd_detail/amd_hip_texture_types.h deleted file mode 100644 index f53362ae9e..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_texture_types.h +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright (c) 2015-2016 Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/hip_texture_types.h - * @brief Defines the different newt vector types for HIP runtime. - */ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_TEXTURE_TYPES_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_TEXTURE_TYPES_H - -/******************************************************************************* - * * - * * - * * - *******************************************************************************/ -#if !defined(__HIPCC_RTC__) -#include -#include -#endif // !defined(__HIPCC_RTC__) -#include - -#if __cplusplus - -/******************************************************************************* - * * - * * - * * - *******************************************************************************/ -#if __HIP__ -#define __HIP_TEXTURE_ATTRIB __attribute__((device_builtin_texture_type)) -#else -#define __HIP_TEXTURE_ATTRIB -#endif - -typedef textureReference* hipTexRef; - -template -struct __HIP_TEXTURE_ATTRIB texture : public textureReference { - texture(int norm = 0, enum hipTextureFilterMode fMode = hipFilterModePoint, - enum hipTextureAddressMode aMode = hipAddressModeClamp) { - normalized = norm; - readMode = mode; - filterMode = fMode; - addressMode[0] = aMode; - addressMode[1] = aMode; - addressMode[2] = aMode; - channelDesc = hipCreateChannelDesc(); - sRGB = 0; - textureObject = nullptr; - maxAnisotropy = 0; - mipmapLevelBias = 0; - minMipmapLevelClamp = 0; - maxMipmapLevelClamp = 0; - } - - texture(int norm, enum hipTextureFilterMode fMode, enum hipTextureAddressMode aMode, - struct hipChannelFormatDesc desc) { - normalized = norm; - readMode = mode; - filterMode = fMode; - addressMode[0] = aMode; - addressMode[1] = aMode; - addressMode[2] = aMode; - channelDesc = desc; - sRGB = 0; - textureObject = nullptr; - maxAnisotropy = 0; - mipmapLevelBias = 0; - minMipmapLevelClamp = 0; - maxMipmapLevelClamp = 0; - } -}; - -#endif /* __cplusplus */ - -#endif /* !HIP_INCLUDE_HIP_AMD_DETAIL_HIP_TEXTURE_TYPES_H */ diff --git a/src/hipamd/include/hip/amd_detail/amd_hip_vector_types.h b/src/hipamd/include/hip/amd_detail/amd_hip_vector_types.h deleted file mode 100644 index c102750531..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hip_vector_types.h +++ /dev/null @@ -1,1598 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/hip_vector_types.h - * @brief Defines the different newt vector types for HIP runtime. - */ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_VECTOR_TYPES_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_VECTOR_TYPES_H - -#include "hip/amd_detail/host_defines.h" - -#if defined(__HIPCC_RTC__) - #define __HOST_DEVICE__ __device__ -#else - #define __HOST_DEVICE__ __host__ __device__ -#endif - -#if defined(__has_attribute) - #if __has_attribute(ext_vector_type) - #define __NATIVE_VECTOR__(n, T) T __attribute__((ext_vector_type(n))) - #else - #define __NATIVE_VECTOR__(n, T) T[n] - #endif - -#if defined(__cplusplus) -// FIXME: Temporarily enable C++ headers while mainline headers is outdated. -//#if !defined(__HIPCC_RTC__) - #include - #include - #include -//#endif // !defined(__HIPCC_RTC__) - - namespace hip_impl { - template struct Scalar_accessor; - } // Namespace hip_impl. - - namespace std { - template - struct is_integral> - : is_integral {}; - template - struct is_floating_point> - : is_floating_point {}; - } // Namespace std. - - namespace hip_impl { - template - struct Scalar_accessor { - struct Address { - const Scalar_accessor* p; - - __HOST_DEVICE__ - operator const T*() const noexcept { - return &reinterpret_cast(p)[idx]; - } - __HOST_DEVICE__ - operator const T*() const volatile noexcept { - return &reinterpret_cast(p)[idx]; - } - __HOST_DEVICE__ - operator T*() noexcept { - return &reinterpret_cast( - const_cast(p))[idx]; - } - __HOST_DEVICE__ - operator T*() volatile noexcept { - return &reinterpret_cast( - const_cast(p))[idx]; - } - }; - - friend - inline - std::ostream& operator<<(std::ostream& os, - const Scalar_accessor& x) noexcept { - return os << x.data[idx]; - } - friend - inline - std::istream& operator>>(std::istream& is, - Scalar_accessor& x) noexcept { - T tmp; - is >> tmp; - x.data[idx] = tmp; - - return is; - } - - // Idea from https://t0rakka.silvrback.com/simd-scalar-accessor - Vector data; - - __HOST_DEVICE__ - operator T() const noexcept { return data[idx]; } - __HOST_DEVICE__ - operator T() const volatile noexcept { return data[idx]; } - -#ifdef __HIP_ENABLE_VECTOR_SCALAR_ACCESSORY_ENUM_CONVERSION__ - // The conversions to enum are fairly ghastly, but unfortunately used in - // some pre-existing, difficult to modify, code. - template< - typename U, - typename std::enable_if< - !std::is_same{} && - std::is_enum{} && - std::is_convertible< - T, typename std::enable_if::value, std::underlying_type>::type::type>{}>::type* = nullptr> - __HOST_DEVICE__ - operator U() const noexcept { return static_cast(data[idx]); } - template< - typename U, - typename std::enable_if< - !std::is_same{} && - std::is_enum{} && - std::is_convertible< - T, typename std::enable_if::value, std::underlying_type>::type::type>{}>::type* = nullptr> - __HOST_DEVICE__ - operator U() const volatile noexcept { return static_cast(data[idx]); } -#endif - - __HOST_DEVICE__ - operator T&() noexcept { - return reinterpret_cast< - T (&)[sizeof(Vector) / sizeof(T)]>(data)[idx]; - } - __HOST_DEVICE__ - operator volatile T&() volatile noexcept { - return reinterpret_cast< - volatile T (&)[sizeof(Vector) / sizeof(T)]>(data)[idx]; - } - - __HOST_DEVICE__ - Address operator&() const noexcept { return Address{this}; } - - __HOST_DEVICE__ - Scalar_accessor& operator=(const Scalar_accessor& x) noexcept { - data[idx] = x.data[idx]; - - return *this; - } - __HOST_DEVICE__ - Scalar_accessor& operator=(T x) noexcept { - data[idx] = x; - - return *this; - } - __HOST_DEVICE__ - volatile Scalar_accessor& operator=(T x) volatile noexcept { - data[idx] = x; - - return *this; - } - - __HOST_DEVICE__ - Scalar_accessor& operator++() noexcept { - ++data[idx]; - return *this; - } - __HOST_DEVICE__ - T operator++(int) noexcept { - auto r{data[idx]}; - ++data[idx]; - return *this; - } - __HOST_DEVICE__ - Scalar_accessor& operator--() noexcept { - --data[idx]; - return *this; - } - __HOST_DEVICE__ - T operator--(int) noexcept { - auto r{data[idx]}; - --data[idx]; - return *this; - } - - // TODO: convertibility is too restrictive, constraint should be on - // the operator being invocable with a value of type U. - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator+=(U x) noexcept { - data[idx] += x; - return *this; - } - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator-=(U x) noexcept { - data[idx] -= x; - return *this; - } - - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator*=(U x) noexcept { - data[idx] *= x; - return *this; - } - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator/=(U x) noexcept { - data[idx] /= x; - return *this; - } - template< - typename U = T, - typename std::enable_if{} && - std::is_integral{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator%=(U x) noexcept { - data[idx] %= x; - return *this; - } - - template< - typename U = T, - typename std::enable_if{} && - std::is_integral{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator>>=(U x) noexcept { - data[idx] >>= x; - return *this; - } - template< - typename U = T, - typename std::enable_if{} && - std::is_integral{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator<<=(U x) noexcept { - data[idx] <<= x; - return *this; - } - template< - typename U = T, - typename std::enable_if{} && - std::is_integral{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator&=(U x) noexcept { - data[idx] &= x; - return *this; - } - template< - typename U = T, - typename std::enable_if{} && - std::is_integral{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator|=(U x) noexcept { - data[idx] |= x; - return *this; - } - template< - typename U = T, - typename std::enable_if{} && - std::is_integral{}>::type* = nullptr> - __HOST_DEVICE__ - Scalar_accessor& operator^=(U x) noexcept { - data[idx] ^= x; - return *this; - } - }; - - inline - constexpr - unsigned int next_pot(unsigned int x) { - // Precondition: x > 1. - return 1u << (32u - __builtin_clz(x - 1u)); - } - } // Namespace hip_impl. - - template struct HIP_vector_base; - - template - struct HIP_vector_base { - using Native_vec_ = __NATIVE_VECTOR__(1, T); - - union { - Native_vec_ data; -#if __HIP_CLANG_ONLY__ - struct { - T x; - }; -#else - hip_impl::Scalar_accessor x; -#endif - }; - - using value_type = T; - - __HOST_DEVICE__ - HIP_vector_base() = default; - __HOST_DEVICE__ - explicit - constexpr - HIP_vector_base(T x_) noexcept : data{x_} {} - __HOST_DEVICE__ - constexpr - HIP_vector_base(const HIP_vector_base&) = default; - __HOST_DEVICE__ - constexpr - HIP_vector_base(HIP_vector_base&&) = default; - __HOST_DEVICE__ - ~HIP_vector_base() = default; - - __HOST_DEVICE__ - HIP_vector_base& operator=(const HIP_vector_base& x_) noexcept { - #if __has_attribute(ext_vector_type) - data = x_.data; - #else - data[0] = x_.data[0]; - #endif - - return *this; - } - }; - - template - struct HIP_vector_base { - using Native_vec_ = __NATIVE_VECTOR__(2, T); - - union - #if !__has_attribute(ext_vector_type) - alignas(hip_impl::next_pot(2 * sizeof(T))) - #endif - { - Native_vec_ data; -#if __HIP_CLANG_ONLY__ - struct { - T x; - T y; - }; -#else - hip_impl::Scalar_accessor x; - hip_impl::Scalar_accessor y; -#endif - }; - - using value_type = T; - - __HOST_DEVICE__ - HIP_vector_base() = default; - __HOST_DEVICE__ - explicit - constexpr - HIP_vector_base(T x_) noexcept : data{x_, x_} {} - __HOST_DEVICE__ - constexpr - HIP_vector_base(T x_, T y_) noexcept : data{x_, y_} {} - __HOST_DEVICE__ - constexpr - HIP_vector_base(const HIP_vector_base&) = default; - __HOST_DEVICE__ - constexpr - HIP_vector_base(HIP_vector_base&&) = default; - __HOST_DEVICE__ - ~HIP_vector_base() = default; - - __HOST_DEVICE__ - HIP_vector_base& operator=(const HIP_vector_base& x_) noexcept { - #if __has_attribute(ext_vector_type) - data = x_.data; - #else - data[0] = x_.data[0]; - data[1] = x_.data[1]; - #endif - - return *this; - } - }; - - template - struct HIP_vector_base { - struct Native_vec_ { - T d[3]; - - __HOST_DEVICE__ - Native_vec_() = default; - - __HOST_DEVICE__ - explicit - constexpr - Native_vec_(T x_) noexcept : d{x_, x_, x_} {} - __HOST_DEVICE__ - constexpr - Native_vec_(T x_, T y_, T z_) noexcept : d{x_, y_, z_} {} - __HOST_DEVICE__ - constexpr - Native_vec_(const Native_vec_&) = default; - __HOST_DEVICE__ - constexpr - Native_vec_(Native_vec_&&) = default; - __HOST_DEVICE__ - ~Native_vec_() = default; - - __HOST_DEVICE__ - Native_vec_& operator=(const Native_vec_&) = default; - __HOST_DEVICE__ - Native_vec_& operator=(Native_vec_&&) = default; - - __HOST_DEVICE__ - T& operator[](unsigned int idx) noexcept { return d[idx]; } - __HOST_DEVICE__ - T operator[](unsigned int idx) const noexcept { return d[idx]; } - - __HOST_DEVICE__ - Native_vec_& operator+=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] += x_.d[i]; - return *this; - } - __HOST_DEVICE__ - Native_vec_& operator-=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] -= x_.d[i]; - return *this; - } - - __HOST_DEVICE__ - Native_vec_& operator*=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] *= x_.d[i]; - return *this; - } - __HOST_DEVICE__ - Native_vec_& operator/=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] /= x_.d[i]; - return *this; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - Native_vec_ operator-() const noexcept - { - auto r{*this}; - for (auto&& x : r.d) x = -x; - return r; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - Native_vec_ operator~() const noexcept - { - auto r{*this}; - for (auto&& x : r.d) x = ~x; - return r; - } - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - Native_vec_& operator%=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] %= x_.d[i]; - return *this; - } - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - Native_vec_& operator^=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] ^= x_.d[i]; - return *this; - } - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - Native_vec_& operator|=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] |= x_.d[i]; - return *this; - } - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - Native_vec_& operator&=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] &= x_.d[i]; - return *this; - } - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - Native_vec_& operator>>=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] >>= x_.d[i]; - return *this; - } - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - Native_vec_& operator<<=(const Native_vec_& x_) noexcept - { - for (auto i = 0u; i != 3u; ++i) d[i] <<= x_.d[i]; - return *this; - } - - using Vec3_cmp = int __attribute__((vector_size(4 * sizeof(int)))); - __HOST_DEVICE__ - Vec3_cmp operator==(const Native_vec_& x_) const noexcept - { - return Vec3_cmp{d[0] == x_.d[0], d[1] == x_.d[1], d[2] == x_.d[2]}; - } - }; - - union { - Native_vec_ data; - struct { - T x; - T y; - T z; - }; - }; - - using value_type = T; - - __HOST_DEVICE__ - HIP_vector_base() = default; - __HOST_DEVICE__ - explicit - constexpr - HIP_vector_base(T x_) noexcept : data{x_, x_, x_} {} - __HOST_DEVICE__ - constexpr - HIP_vector_base(T x_, T y_, T z_) noexcept : data{x_, y_, z_} {} - __HOST_DEVICE__ - constexpr - HIP_vector_base(const HIP_vector_base&) = default; - __HOST_DEVICE__ - constexpr - HIP_vector_base(HIP_vector_base&&) = default; - __HOST_DEVICE__ - ~HIP_vector_base() = default; - - __HOST_DEVICE__ - HIP_vector_base& operator=(const HIP_vector_base&) = default; - __HOST_DEVICE__ - HIP_vector_base& operator=(HIP_vector_base&&) = default; - }; - - template - struct HIP_vector_base { - using Native_vec_ = __NATIVE_VECTOR__(4, T); - - union - #if !__has_attribute(ext_vector_type) - alignas(hip_impl::next_pot(4 * sizeof(T))) - #endif - { - Native_vec_ data; -#if __HIP_CLANG_ONLY__ - struct { - T x; - T y; - T z; - T w; - }; -#else - hip_impl::Scalar_accessor x; - hip_impl::Scalar_accessor y; - hip_impl::Scalar_accessor z; - hip_impl::Scalar_accessor w; -#endif - }; - - using value_type = T; - - __HOST_DEVICE__ - HIP_vector_base() = default; - __HOST_DEVICE__ - explicit - constexpr - HIP_vector_base(T x_) noexcept : data{x_, x_, x_, x_} {} - __HOST_DEVICE__ - constexpr - HIP_vector_base(T x_, T y_, T z_, T w_) noexcept : data{x_, y_, z_, w_} {} - __HOST_DEVICE__ - constexpr - HIP_vector_base(const HIP_vector_base&) = default; - __HOST_DEVICE__ - constexpr - HIP_vector_base(HIP_vector_base&&) = default; - __HOST_DEVICE__ - ~HIP_vector_base() = default; - - __HOST_DEVICE__ - HIP_vector_base& operator=(const HIP_vector_base& x_) noexcept { - #if __has_attribute(ext_vector_type) - data = x_.data; - #else - data[0] = x_.data[0]; - data[1] = x_.data[1]; - data[2] = x_.data[2]; - data[3] = x_.data[3]; - #endif - - return *this; - } - }; - - template - struct HIP_vector_type : public HIP_vector_base { - using HIP_vector_base::data; - using typename HIP_vector_base::Native_vec_; - - __HOST_DEVICE__ - HIP_vector_type() = default; - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - explicit - constexpr - HIP_vector_type(U x_) noexcept - : HIP_vector_base{static_cast(x_)} - {} - template< // TODO: constrain based on type as well. - typename... Us, - typename std::enable_if< - (rank > 1) && sizeof...(Us) == rank>::type* = nullptr> - __HOST_DEVICE__ - constexpr - HIP_vector_type(Us... xs) noexcept - : HIP_vector_base{static_cast(xs)...} - {} - __HOST_DEVICE__ - constexpr - HIP_vector_type(const HIP_vector_type&) = default; - __HOST_DEVICE__ - constexpr - HIP_vector_type(HIP_vector_type&&) = default; - __HOST_DEVICE__ - ~HIP_vector_type() = default; - - __HOST_DEVICE__ - HIP_vector_type& operator=(const HIP_vector_type&) = default; - __HOST_DEVICE__ - HIP_vector_type& operator=(HIP_vector_type&&) = default; - - // Operators - __HOST_DEVICE__ - HIP_vector_type& operator++() noexcept - { - return *this += HIP_vector_type{1}; - } - __HOST_DEVICE__ - HIP_vector_type operator++(int) noexcept - { - auto tmp(*this); - ++*this; - return tmp; - } - - __HOST_DEVICE__ - HIP_vector_type& operator--() noexcept - { - return *this -= HIP_vector_type{1}; - } - __HOST_DEVICE__ - HIP_vector_type operator--(int) noexcept - { - auto tmp(*this); - --*this; - return tmp; - } - - __HOST_DEVICE__ - HIP_vector_type& operator+=(const HIP_vector_type& x) noexcept - { - data += x.data; - return *this; - } - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator+=(U x) noexcept - { - return *this += HIP_vector_type{x}; - } - - __HOST_DEVICE__ - HIP_vector_type& operator-=(const HIP_vector_type& x) noexcept - { - data -= x.data; - return *this; - } - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator-=(U x) noexcept - { - return *this -= HIP_vector_type{x}; - } - - __HOST_DEVICE__ - HIP_vector_type& operator*=(const HIP_vector_type& x) noexcept - { - data *= x.data; - return *this; - } - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator*=(U x) noexcept - { - return *this *= HIP_vector_type{x}; - } - - __HOST_DEVICE__ - HIP_vector_type& operator/=(const HIP_vector_type& x) noexcept - { - data /= x.data; - return *this; - } - template< - typename U, - typename std::enable_if< - std::is_convertible{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator/=(U x) noexcept - { - return *this /= HIP_vector_type{x}; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type operator-() const noexcept - { - auto tmp(*this); - tmp.data = -tmp.data; - return tmp; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type operator~() const noexcept - { - HIP_vector_type r{*this}; - r.data = ~r.data; - return r; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator%=(const HIP_vector_type& x) noexcept - { - data %= x.data; - return *this; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator^=(const HIP_vector_type& x) noexcept - { - data ^= x.data; - return *this; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator|=(const HIP_vector_type& x) noexcept - { - data |= x.data; - return *this; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator&=(const HIP_vector_type& x) noexcept - { - data &= x.data; - return *this; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator>>=(const HIP_vector_type& x) noexcept - { - data >>= x.data; - return *this; - } - - template< - typename U = T, - typename std::enable_if{}>::type* = nullptr> - __HOST_DEVICE__ - HIP_vector_type& operator<<=(const HIP_vector_type& x) noexcept - { - data <<= x.data; - return *this; - } - }; - - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator+( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} += y; - } - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator+( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} += HIP_vector_type{y}; - } - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator+( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} += y; - } - - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator-( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} -= y; - } - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator-( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} -= HIP_vector_type{y}; - } - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator-( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} -= y; - } - - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator*( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} *= y; - } - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator*( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} *= HIP_vector_type{y}; - } - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator*( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} *= y; - } - - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator/( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} /= y; - } - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator/( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} /= HIP_vector_type{y}; - } - template - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator/( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} /= y; - } - - template - __HOST_DEVICE__ - inline - constexpr - bool _hip_any_zero(const V& x, int n) noexcept - { - return - (n == -1) ? true : ((x[n] == 0) ? false : _hip_any_zero(x, n - 1)); - } - - template - __HOST_DEVICE__ - inline - constexpr - bool operator==( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return _hip_any_zero(x.data == y.data, n - 1); - } - template - __HOST_DEVICE__ - inline - constexpr - bool operator==(const HIP_vector_type& x, U y) noexcept - { - return x == HIP_vector_type{y}; - } - template - __HOST_DEVICE__ - inline - constexpr - bool operator==(U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} == y; - } - - template - __HOST_DEVICE__ - inline - constexpr - bool operator!=( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return !(x == y); - } - template - __HOST_DEVICE__ - inline - constexpr - bool operator!=(const HIP_vector_type& x, U y) noexcept - { - return !(x == y); - } - template - __HOST_DEVICE__ - inline - constexpr - bool operator!=(U x, const HIP_vector_type& y) noexcept - { - return !(x == y); - } - - template< - typename T, - unsigned int n, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator%( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} %= y; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator%( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} %= HIP_vector_type{y}; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator%( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} %= y; - } - - template< - typename T, - unsigned int n, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator^( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} ^= y; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator^( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} ^= HIP_vector_type{y}; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator^( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} ^= y; - } - - template< - typename T, - unsigned int n, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator|( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} |= y; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator|( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} |= HIP_vector_type{y}; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator|( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} |= y; - } - - template< - typename T, - unsigned int n, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator&( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} &= y; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator&( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} &= HIP_vector_type{y}; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator&( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} &= y; - } - - template< - typename T, - unsigned int n, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator>>( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} >>= y; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator>>( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} >>= HIP_vector_type{y}; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator>>( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} >>= y; - } - - template< - typename T, - unsigned int n, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator<<( - const HIP_vector_type& x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} <<= y; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator<<( - const HIP_vector_type& x, U y) noexcept - { - return HIP_vector_type{x} <<= HIP_vector_type{y}; - } - template< - typename T, - unsigned int n, - typename U, - typename std::enable_if::value>::type, - typename std::enable_if{}>* = nullptr> - __HOST_DEVICE__ - inline - constexpr - HIP_vector_type operator<<( - U x, const HIP_vector_type& y) noexcept - { - return HIP_vector_type{x} <<= y; - } - - #define __MAKE_VECTOR_TYPE__(CUDA_name, T) \ - using CUDA_name##1 = HIP_vector_type;\ - using CUDA_name##2 = HIP_vector_type;\ - using CUDA_name##3 = HIP_vector_type;\ - using CUDA_name##4 = HIP_vector_type; -#else - #define __MAKE_VECTOR_TYPE__(CUDA_name, T) \ - typedef struct {\ - T x;\ - } CUDA_name##1;\ - typedef struct {\ - T x;\ - T y;\ - } CUDA_name##2;\ - typedef struct {\ - T x;\ - T y;\ - T z;\ - } CUDA_name##3;\ - typedef struct {\ - T x;\ - T y;\ - T z;\ - T w;\ - } CUDA_name##4; -#endif - -__MAKE_VECTOR_TYPE__(uchar, unsigned char); -__MAKE_VECTOR_TYPE__(char, char); -__MAKE_VECTOR_TYPE__(ushort, unsigned short); -__MAKE_VECTOR_TYPE__(short, short); -__MAKE_VECTOR_TYPE__(uint, unsigned int); -__MAKE_VECTOR_TYPE__(int, int); -__MAKE_VECTOR_TYPE__(ulong, unsigned long); -__MAKE_VECTOR_TYPE__(long, long); -__MAKE_VECTOR_TYPE__(ulonglong, unsigned long long); -__MAKE_VECTOR_TYPE__(longlong, long long); -__MAKE_VECTOR_TYPE__(float, float); -__MAKE_VECTOR_TYPE__(double, double); - -#ifdef __cplusplus -#define DECLOP_MAKE_ONE_COMPONENT(comp, type) \ - static inline __HOST_DEVICE__ \ - type make_##type(comp x) { type r{x}; return r; } - -#define DECLOP_MAKE_TWO_COMPONENT(comp, type) \ - static inline __HOST_DEVICE__ \ - type make_##type(comp x, comp y) { type r{x, y}; return r; } - -#define DECLOP_MAKE_THREE_COMPONENT(comp, type) \ - static inline __HOST_DEVICE__ \ - type make_##type(comp x, comp y, comp z) { type r{x, y, z}; return r; } - -#define DECLOP_MAKE_FOUR_COMPONENT(comp, type) \ - static inline __HOST_DEVICE__ \ - type make_##type(comp x, comp y, comp z, comp w) { \ - type r{x, y, z, w}; \ - return r; \ - } -#else - #define DECLOP_MAKE_ONE_COMPONENT(comp, type) \ - static inline __HOST_DEVICE__ \ - type make_##type(comp x) { type r; r.x =x; return r; } - - #define DECLOP_MAKE_TWO_COMPONENT(comp, type) \ - static inline __HOST_DEVICE__ \ - type make_##type(comp x, comp y) { type r; r.x=x; r.y=y; return r; } - - #define DECLOP_MAKE_THREE_COMPONENT(comp, type) \ - static inline __HOST_DEVICE__ \ - type make_##type(comp x, comp y, comp z) { type r; r.x=x; r.y=y; r.z=z; return r; } - - #define DECLOP_MAKE_FOUR_COMPONENT(comp, type) \ - static inline __HOST_DEVICE__ \ - type make_##type(comp x, comp y, comp z, comp w) { \ - type r; r.x=x; r.y=y; r.z=z; r.w=w; \ - return r; \ - } -#endif - -DECLOP_MAKE_ONE_COMPONENT(unsigned char, uchar1); -DECLOP_MAKE_TWO_COMPONENT(unsigned char, uchar2); -DECLOP_MAKE_THREE_COMPONENT(unsigned char, uchar3); -DECLOP_MAKE_FOUR_COMPONENT(unsigned char, uchar4); - -DECLOP_MAKE_ONE_COMPONENT(signed char, char1); -DECLOP_MAKE_TWO_COMPONENT(signed char, char2); -DECLOP_MAKE_THREE_COMPONENT(signed char, char3); -DECLOP_MAKE_FOUR_COMPONENT(signed char, char4); - -DECLOP_MAKE_ONE_COMPONENT(unsigned short, ushort1); -DECLOP_MAKE_TWO_COMPONENT(unsigned short, ushort2); -DECLOP_MAKE_THREE_COMPONENT(unsigned short, ushort3); -DECLOP_MAKE_FOUR_COMPONENT(unsigned short, ushort4); - -DECLOP_MAKE_ONE_COMPONENT(signed short, short1); -DECLOP_MAKE_TWO_COMPONENT(signed short, short2); -DECLOP_MAKE_THREE_COMPONENT(signed short, short3); -DECLOP_MAKE_FOUR_COMPONENT(signed short, short4); - -DECLOP_MAKE_ONE_COMPONENT(unsigned int, uint1); -DECLOP_MAKE_TWO_COMPONENT(unsigned int, uint2); -DECLOP_MAKE_THREE_COMPONENT(unsigned int, uint3); -DECLOP_MAKE_FOUR_COMPONENT(unsigned int, uint4); - -DECLOP_MAKE_ONE_COMPONENT(signed int, int1); -DECLOP_MAKE_TWO_COMPONENT(signed int, int2); -DECLOP_MAKE_THREE_COMPONENT(signed int, int3); -DECLOP_MAKE_FOUR_COMPONENT(signed int, int4); - -DECLOP_MAKE_ONE_COMPONENT(float, float1); -DECLOP_MAKE_TWO_COMPONENT(float, float2); -DECLOP_MAKE_THREE_COMPONENT(float, float3); -DECLOP_MAKE_FOUR_COMPONENT(float, float4); - -DECLOP_MAKE_ONE_COMPONENT(double, double1); -DECLOP_MAKE_TWO_COMPONENT(double, double2); -DECLOP_MAKE_THREE_COMPONENT(double, double3); -DECLOP_MAKE_FOUR_COMPONENT(double, double4); - -DECLOP_MAKE_ONE_COMPONENT(unsigned long, ulong1); -DECLOP_MAKE_TWO_COMPONENT(unsigned long, ulong2); -DECLOP_MAKE_THREE_COMPONENT(unsigned long, ulong3); -DECLOP_MAKE_FOUR_COMPONENT(unsigned long, ulong4); - -DECLOP_MAKE_ONE_COMPONENT(signed long, long1); -DECLOP_MAKE_TWO_COMPONENT(signed long, long2); -DECLOP_MAKE_THREE_COMPONENT(signed long, long3); -DECLOP_MAKE_FOUR_COMPONENT(signed long, long4); - -DECLOP_MAKE_ONE_COMPONENT(unsigned long long, ulonglong1); -DECLOP_MAKE_TWO_COMPONENT(unsigned long long, ulonglong2); -DECLOP_MAKE_THREE_COMPONENT(unsigned long long, ulonglong3); -DECLOP_MAKE_FOUR_COMPONENT(unsigned long long, ulonglong4); - -DECLOP_MAKE_ONE_COMPONENT(signed long long, longlong1); -DECLOP_MAKE_TWO_COMPONENT(signed long long, longlong2); -DECLOP_MAKE_THREE_COMPONENT(signed long long, longlong3); -DECLOP_MAKE_FOUR_COMPONENT(signed long long, longlong4); -#else // !defined(__has_attribute) - -#if defined(_MSC_VER) -#include -#include -#include -#include - -typedef union { char data; } char1; -typedef union { char data[2]; } char2; -typedef union { char data[4]; } char4; -typedef union { char4 data; } char3; -typedef union { __m64 data; } char8; -typedef union { __m128i data; } char16; - -typedef union { unsigned char data; } uchar1; -typedef union { unsigned char data[2]; } uchar2; -typedef union { unsigned char data[4]; } uchar4; -typedef union { uchar4 data; } uchar3; -typedef union { __m64 data; } uchar8; -typedef union { __m128i data; } uchar16; - -typedef union { short data; } short1; -typedef union { short data[2]; } short2; -typedef union { __m64 data; } short4; -typedef union { short4 data; } short3; -typedef union { __m128i data; } short8; -typedef union { __m128i data[2]; } short16; - -typedef union { unsigned short data; } ushort1; -typedef union { unsigned short data[2]; } ushort2; -typedef union { __m64 data; } ushort4; -typedef union { ushort4 data; } ushort3; -typedef union { __m128i data; } ushort8; -typedef union { __m128i data[2]; } ushort16; - -typedef union { int data; } int1; -typedef union { __m64 data; } int2; -typedef union { __m128i data; } int4; -typedef union { int4 data; } int3; -typedef union { __m128i data[2]; } int8; -typedef union { __m128i data[4];} int16; - -typedef union { unsigned int data; } uint1; -typedef union { __m64 data; } uint2; -typedef union { __m128i data; } uint4; -typedef union { uint4 data; } uint3; -typedef union { __m128i data[2]; } uint8; -typedef union { __m128i data[4]; } uint16; - -#if !defined(_WIN64) -typedef union { int data; } long1; -typedef union { __m64 data; } long2; -typedef union { __m128i data; } long4; -typedef union { long4 data; } long3; -typedef union { __m128i data[2]; } long8; -typedef union { __m128i data[4]; } long16; - -typedef union { unsigned int data; } ulong1; -typedef union { __m64 data; } ulong2; -typedef union { __m128i data; } ulong4; -typedef union { ulong4 data; } ulong3; -typedef union { __m128i data[2]; } ulong8; -typedef union { __m128i data[4]; } ulong16; -#else // defined(_WIN64) -typedef union { __m64 data; } long1; -typedef union { __m128i data; } long2; -typedef union { __m128i data[2]; } long4; -typedef union { long4 data; } long3; -typedef union { __m128i data[4]; } long8; -typedef union { __m128i data[8]; } long16; - -typedef union { __m64 data; } ulong1; -typedef union { __m128i data; } ulong2; -typedef union { __m128i data[2]; } ulong4; -typedef union { ulong4 data; } ulong3; -typedef union { __m128i data[4]; } ulong8; -typedef union { __m128i data[8]; } ulong16; -#endif // defined(_WIN64) - -typedef union { __m64 data; } longlong1; -typedef union { __m128i data; } longlong2; -typedef union { __m128i data[2]; } longlong4; -typedef union { longlong4 data; } longlong3; -typedef union { __m128i data[4]; } longlong8; -typedef union { __m128i data[8]; } longlong16; - -typedef union { __m64 data; } ulonglong1; -typedef union { __m128i data; } ulonglong2; -typedef union { __m128i data[2]; } ulonglong4; -typedef union { ulonglong4 data; } ulonglong3; -typedef union { __m128i data[4]; } ulonglong8; -typedef union { __m128i data[8]; } ulonglong16; - -typedef union { float data; } float1; -typedef union { __m64 data; } float2; -typedef union { __m128 data; } float4; -typedef union { float4 data; } float3; -typedef union { __m256 data; } float8; -typedef union { __m256 data[2]; } float16; - -typedef union { double data; } double1; -typedef union { __m128d data; } double2; -typedef union { __m256d data; } double4; -typedef union { double4 data; } double3; -typedef union { __m256d data[2]; } double8; -typedef union { __m256d data[4]; } double16; - -#else // !defined(_MSC_VER) - -typedef union { char data; } char1; -typedef union { char data[2]; } char2; -typedef union { char data[4]; } char4; -typedef union { char data[8]; } char8; -typedef union { char data[16]; } char16; -typedef union { char4 data; } char3; - -typedef union { unsigned char data; } uchar1; -typedef union { unsigned char data[2]; } uchar2; -typedef union { unsigned char data[4]; } uchar4; -typedef union { unsigned char data[8]; } uchar8; -typedef union { unsigned char data[16]; } uchar16; -typedef union { uchar4 data; } uchar3; - -typedef union { short data; } short1; -typedef union { short data[2]; } short2; -typedef union { short data[4]; } short4; -typedef union { short data[8]; } short8; -typedef union { short data[16]; } short16; -typedef union { short4 data; } short3; - -typedef union { unsigned short data; } ushort1; -typedef union { unsigned short data[2]; } ushort2; -typedef union { unsigned short data[4]; } ushort4; -typedef union { unsigned short data[8]; } ushort8; -typedef union { unsigned short data[16]; } ushort16; -typedef union { ushort4 data; } ushort3; - -typedef union { int data; } int1; -typedef union { int data[2]; } int2; -typedef union { int data[4]; } int4; -typedef union { int data[8]; } int8; -typedef union { int data[16]; } int16; -typedef union { int4 data; } int3; - -typedef union { unsigned int data; } uint1; -typedef union { unsigned int data[2]; } uint2; -typedef union { unsigned int data[4]; } uint4; -typedef union { unsigned int data[8]; } uint8; -typedef union { unsigned int data[16]; } uint16; -typedef union { uint4 data; } uint3; - -typedef union { long data; } long1; -typedef union { long data[2]; } long2; -typedef union { long data[4]; } long4; -typedef union { long data[8]; } long8; -typedef union { long data[16]; } long16; -typedef union { long4 data; } long3; - -typedef union { unsigned long data; } ulong1; -typedef union { unsigned long data[2]; } ulong2; -typedef union { unsigned long data[4]; } ulong4; -typedef union { unsigned long data[8]; } ulong8; -typedef union { unsigned long data[16]; } ulong16; -typedef union { ulong4 data; } ulong3; - -typedef union { long long data; } longlong1; -typedef union { long long data[2]; } longlong2; -typedef union { long long data[4]; } longlong4; -typedef union { long long data[8]; } longlong8; -typedef union { long long data[16]; } longlong16; -typedef union { longlong4 data; } longlong3; - -typedef union { unsigned long long data; } ulonglong1; -typedef union { unsigned long long data[2]; } ulonglong2; -typedef union { unsigned long long data[4]; } ulonglong4; -typedef union { unsigned long long data[8]; } ulonglong8; -typedef union { unsigned long long data[16]; } ulonglong16; -typedef union { ulonglong4 data; } ulonglong3; - -typedef union { float data; } float1; -typedef union { float data[2]; } float2; -typedef union { float data[4]; } float4; -typedef union { float data[8]; } float8; -typedef union { float data[16]; } float16; -typedef union { float4 data; } float3; - -typedef union { double data; } double1; -typedef union { double data[2]; } double2; -typedef union { double data[4]; } double4; -typedef union { double data[8]; } double8; -typedef union { double data[16]; } double16; -typedef union { double4 data; } double3; - -#endif // defined(_MSC_VER) -#endif // defined(__has_attribute) -#endif diff --git a/src/hipamd/include/hip/amd_detail/amd_hiprtc.h b/src/hipamd/include/hip/amd_detail/amd_hiprtc.h deleted file mode 100644 index fecea75340..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_hiprtc.h +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ -#ifndef HIPRTC_H -#define HIPRTC_H - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#include - -#if !defined(_WIN32) -#pragma GCC visibility push (default) -#endif - -enum hiprtcResult { - HIPRTC_SUCCESS = 0, - HIPRTC_ERROR_OUT_OF_MEMORY = 1, - HIPRTC_ERROR_PROGRAM_CREATION_FAILURE = 2, - HIPRTC_ERROR_INVALID_INPUT = 3, - HIPRTC_ERROR_INVALID_PROGRAM = 4, - HIPRTC_ERROR_INVALID_OPTION = 5, - HIPRTC_ERROR_COMPILATION = 6, - HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE = 7, - HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION = 8, - HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION = 9, - HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID = 10, - HIPRTC_ERROR_INTERNAL_ERROR = 11 -}; - -const char* hiprtcGetErrorString(hiprtcResult result); - - -hiprtcResult hiprtcVersion(int* major, int* minor); - -typedef struct _hiprtcProgram* hiprtcProgram; - -hiprtcResult hiprtcAddNameExpression(hiprtcProgram prog, - const char* name_expression); - -hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, - int numOptions, - const char** options); - -hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, - const char* src, - const char* name, - int numHeaders, - const char** headers, - const char** includeNames); - -hiprtcResult hiprtcDestroyProgram(hiprtcProgram* prog); - -hiprtcResult hiprtcGetLoweredName(hiprtcProgram prog, - const char* name_expression, - const char** lowered_name); - -hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* log); - -hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog, - size_t* logSizeRet); - -hiprtcResult hiprtcGetCode(hiprtcProgram prog, char* code); - -hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, size_t* codeSizeRet); - -#if !defined(_WIN32) -#pragma GCC visibility pop -#endif - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif //HIPRTC_H diff --git a/src/hipamd/include/hip/amd_detail/amd_library_types.h b/src/hipamd/include/hip/amd_detail/amd_library_types.h deleted file mode 100644 index 6c3e111220..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_library_types.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_LIBRARY_TYPES_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_LIBRARY_TYPES_H - -typedef enum hipDataType { - HIP_R_16F = 2, - HIP_R_32F = 0, - HIP_R_64F = 1, - HIP_C_16F = 6, - HIP_C_32F = 4, - HIP_C_64F = 5 -} hipDataType; - -typedef enum hipLibraryPropertyType { - HIP_LIBRARY_MAJOR_VERSION, - HIP_LIBRARY_MINOR_VERSION, - HIP_LIBRARY_PATCH_LEVEL -} hipLibraryPropertyType; - -#endif diff --git a/src/hipamd/include/hip/amd_detail/amd_math_functions.h b/src/hipamd/include/hip/amd_detail/amd_math_functions.h deleted file mode 100644 index 24f6433b53..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_math_functions.h +++ /dev/null @@ -1,1502 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#include "hip_fp16_math_fwd.h" -#include "amd_hip_vector_types.h" -#include "math_fwd.h" - -#include - -#if !defined(__HIPCC_RTC__) -#include -// assert.h is only for the host version of assert. -// The device version of assert is implemented in hip/amd_detail/hip_runtime.h. -// Users should include hip_runtime.h for the device version of assert. -#if !__HIP_DEVICE_COMPILE__ -#include -#endif -#include -#include -#include -#endif // !defined(__HIPCC_RTC__) - -#if _LIBCPP_VERSION && __HIP__ -namespace std { -template <> -struct __numeric_type<_Float16> -{ - static _Float16 __test(_Float16); - - typedef _Float16 type; - static const bool value = true; -}; -} -#endif // _LIBCPP_VERSION - -#pragma push_macro("__DEVICE__") -#pragma push_macro("__RETURN_TYPE") - -#define __DEVICE__ static __device__ -#define __RETURN_TYPE bool - -#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -__DEVICE__ -inline -uint64_t __make_mantissa_base8(const char* tagp) -{ - uint64_t r = 0; - while (tagp) { - char tmp = *tagp; - - if (tmp >= '0' && tmp <= '7') r = (r * 8u) + tmp - '0'; - else return 0; - - ++tagp; - } - - return r; -} - -__DEVICE__ -inline -uint64_t __make_mantissa_base10(const char* tagp) -{ - uint64_t r = 0; - while (tagp) { - char tmp = *tagp; - - if (tmp >= '0' && tmp <= '9') r = (r * 10u) + tmp - '0'; - else return 0; - - ++tagp; - } - - return r; -} - -__DEVICE__ -inline -uint64_t __make_mantissa_base16(const char* tagp) -{ - uint64_t r = 0; - while (tagp) { - char tmp = *tagp; - - if (tmp >= '0' && tmp <= '9') r = (r * 16u) + tmp - '0'; - else if (tmp >= 'a' && tmp <= 'f') r = (r * 16u) + tmp - 'a' + 10; - else if (tmp >= 'A' && tmp <= 'F') r = (r * 16u) + tmp - 'A' + 10; - else return 0; - - ++tagp; - } - - return r; -} - -__DEVICE__ -inline -uint64_t __make_mantissa(const char* tagp) -{ - if (!tagp) return 0u; - - if (*tagp == '0') { - ++tagp; - - if (*tagp == 'x' || *tagp == 'X') return __make_mantissa_base16(tagp); - else return __make_mantissa_base8(tagp); - } - - return __make_mantissa_base10(tagp); -} -#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ - -// DOT FUNCTIONS -#if __HIP_CLANG_ONLY__ -__DEVICE__ -inline -int amd_mixed_dot(short2 a, short2 b, int c, bool saturate) { - return __ockl_sdot2(a.data, b.data, c, saturate); -} -__DEVICE__ -inline -uint amd_mixed_dot(ushort2 a, ushort2 b, uint c, bool saturate) { - return __ockl_udot2(a.data, b.data, c, saturate); -} -__DEVICE__ -inline -int amd_mixed_dot(char4 a, char4 b, int c, bool saturate) { - return __ockl_sdot4(a.data, b.data, c, saturate); -} -__DEVICE__ -inline -uint amd_mixed_dot(uchar4 a, uchar4 b, uint c, bool saturate) { - return __ockl_udot4(a.data, b.data, c, saturate); -} -__DEVICE__ -inline -int amd_mixed_dot(int a, int b, int c, bool saturate) { - return __ockl_sdot8(a, b, c, saturate); -} -__DEVICE__ -inline -uint amd_mixed_dot(uint a, uint b, uint c, bool saturate) { - return __ockl_udot8(a, b, c, saturate); -} -#endif - -#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -// BEGIN FLOAT -__DEVICE__ -inline -float abs(float x) { return __ocml_fabs_f32(x); } -__DEVICE__ -inline -float acosf(float x) { return __ocml_acos_f32(x); } -__DEVICE__ -inline -float acoshf(float x) { return __ocml_acosh_f32(x); } -__DEVICE__ -inline -float asinf(float x) { return __ocml_asin_f32(x); } -__DEVICE__ -inline -float asinhf(float x) { return __ocml_asinh_f32(x); } -__DEVICE__ -inline -float atan2f(float x, float y) { return __ocml_atan2_f32(x, y); } -__DEVICE__ -inline -float atanf(float x) { return __ocml_atan_f32(x); } -__DEVICE__ -inline -float atanhf(float x) { return __ocml_atanh_f32(x); } -__DEVICE__ -inline -float cbrtf(float x) { return __ocml_cbrt_f32(x); } -__DEVICE__ -inline -float ceilf(float x) { return __ocml_ceil_f32(x); } -__DEVICE__ -inline -float copysignf(float x, float y) { return __ocml_copysign_f32(x, y); } -__DEVICE__ -inline -float cosf(float x) { return __ocml_cos_f32(x); } -__DEVICE__ -inline -float coshf(float x) { return __ocml_cosh_f32(x); } -__DEVICE__ -inline -float cospif(float x) { return __ocml_cospi_f32(x); } -__DEVICE__ -inline -float cyl_bessel_i0f(float x) { return __ocml_i0_f32(x); } -__DEVICE__ -inline -float cyl_bessel_i1f(float x) { return __ocml_i1_f32(x); } -__DEVICE__ -inline -float erfcf(float x) { return __ocml_erfc_f32(x); } -__DEVICE__ -inline -float erfcinvf(float x) { return __ocml_erfcinv_f32(x); } -__DEVICE__ -inline -float erfcxf(float x) { return __ocml_erfcx_f32(x); } -__DEVICE__ -inline -float erff(float x) { return __ocml_erf_f32(x); } -__DEVICE__ -inline -float erfinvf(float x) { return __ocml_erfinv_f32(x); } -__DEVICE__ -inline -float exp10f(float x) { return __ocml_exp10_f32(x); } -__DEVICE__ -inline -float exp2f(float x) { return __ocml_exp2_f32(x); } -__DEVICE__ -inline -float expf(float x) { return __ocml_exp_f32(x); } -__DEVICE__ -inline -float expm1f(float x) { return __ocml_expm1_f32(x); } -__DEVICE__ -inline -float fabsf(float x) { return __ocml_fabs_f32(x); } -__DEVICE__ -inline -float fdimf(float x, float y) { return __ocml_fdim_f32(x, y); } -__DEVICE__ -inline -float fdividef(float x, float y) { return x / y; } -__DEVICE__ -inline -float floorf(float x) { return __ocml_floor_f32(x); } -__DEVICE__ -inline -float fmaf(float x, float y, float z) { return __ocml_fma_f32(x, y, z); } -__DEVICE__ -inline -float fmaxf(float x, float y) { return __ocml_fmax_f32(x, y); } -__DEVICE__ -inline -float fminf(float x, float y) { return __ocml_fmin_f32(x, y); } -__DEVICE__ -inline -float fmodf(float x, float y) { return __ocml_fmod_f32(x, y); } -__DEVICE__ -inline -float frexpf(float x, int* nptr) -{ - int tmp; - float r = - __ocml_frexp_f32(x, (__attribute__((address_space(5))) int*) &tmp); - *nptr = tmp; - - return r; -} -__DEVICE__ -inline -float hypotf(float x, float y) { return __ocml_hypot_f32(x, y); } -__DEVICE__ -inline -int ilogbf(float x) { return __ocml_ilogb_f32(x); } -__DEVICE__ -inline -__RETURN_TYPE isfinite(float x) { return __ocml_isfinite_f32(x); } -__DEVICE__ -inline -__RETURN_TYPE isinf(float x) { return __ocml_isinf_f32(x); } -__DEVICE__ -inline -__RETURN_TYPE isnan(float x) { return __ocml_isnan_f32(x); } -__DEVICE__ -inline -float j0f(float x) { return __ocml_j0_f32(x); } -__DEVICE__ -inline -float j1f(float x) { return __ocml_j1_f32(x); } -__DEVICE__ -inline -float jnf(int n, float x) -{ // TODO: we could use Ahmes multiplication and the Miller & Brown algorithm - // for linear recurrences to get O(log n) steps, but it's unclear if - // it'd be beneficial in this case. - if (n == 0) return j0f(x); - if (n == 1) return j1f(x); - - float x0 = j0f(x); - float x1 = j1f(x); - for (int i = 1; i < n; ++i) { - float x2 = (2 * i) / x * x1 - x0; - x0 = x1; - x1 = x2; - } - - return x1; -} -__DEVICE__ -inline -float ldexpf(float x, int e) { return __ocml_ldexp_f32(x, e); } -__DEVICE__ -inline -float lgammaf(float x) { return __ocml_lgamma_f32(x); } -__DEVICE__ -inline -long long int llrintf(float x) { return __ocml_rint_f32(x); } -__DEVICE__ -inline -long long int llroundf(float x) { return __ocml_round_f32(x); } -__DEVICE__ -inline -float log10f(float x) { return __ocml_log10_f32(x); } -__DEVICE__ -inline -float log1pf(float x) { return __ocml_log1p_f32(x); } -__DEVICE__ -inline -float log2f(float x) { return __ocml_log2_f32(x); } -__DEVICE__ -inline -float logbf(float x) { return __ocml_logb_f32(x); } -__DEVICE__ -inline -float logf(float x) { return __ocml_log_f32(x); } -__DEVICE__ -inline -long int lrintf(float x) { return __ocml_rint_f32(x); } -__DEVICE__ -inline -long int lroundf(float x) { return __ocml_round_f32(x); } -__DEVICE__ -inline -float modff(float x, float* iptr) -{ - float tmp; - float r = - __ocml_modf_f32(x, (__attribute__((address_space(5))) float*) &tmp); - *iptr = tmp; - - return r; -} -__DEVICE__ -inline -float nanf(const char* tagp) -{ - union { - float val; - struct ieee_float { - uint32_t mantissa : 22; - uint32_t quiet : 1; - uint32_t exponent : 8; - uint32_t sign : 1; - } bits; - - static_assert(sizeof(float) == sizeof(ieee_float), ""); - } tmp; - - tmp.bits.sign = 0u; - tmp.bits.exponent = ~0u; - tmp.bits.quiet = 1u; - tmp.bits.mantissa = __make_mantissa(tagp); - - return tmp.val; -} -__DEVICE__ -inline -float nearbyintf(float x) { return __ocml_nearbyint_f32(x); } -__DEVICE__ -inline -float nextafterf(float x, float y) { return __ocml_nextafter_f32(x, y); } -__DEVICE__ -inline -float norm3df(float x, float y, float z) { return __ocml_len3_f32(x, y, z); } -__DEVICE__ -inline -float norm4df(float x, float y, float z, float w) -{ - return __ocml_len4_f32(x, y, z, w); -} -__DEVICE__ -inline -float normcdff(float x) { return __ocml_ncdf_f32(x); } -__DEVICE__ -inline -float normcdfinvf(float x) { return __ocml_ncdfinv_f32(x); } -__DEVICE__ -inline -float normf(int dim, const float* a) -{ // TODO: placeholder until OCML adds support. - float r = 0; - while (dim--) { r += a[0] * a[0]; ++a; } - - return __ocml_sqrt_f32(r); -} -__DEVICE__ -inline -float powf(float x, float y) { return __ocml_pow_f32(x, y); } -__DEVICE__ -inline -float powif(float base, int iexp) { return __ocml_pown_f32(base, iexp); } -__DEVICE__ -inline -float rcbrtf(float x) { return __ocml_rcbrt_f32(x); } -__DEVICE__ -inline -float remainderf(float x, float y) { return __ocml_remainder_f32(x, y); } -__DEVICE__ -inline -float remquof(float x, float y, int* quo) -{ - int tmp; - float r = - __ocml_remquo_f32(x, y, (__attribute__((address_space(5))) int*) &tmp); - *quo = tmp; - - return r; -} -__DEVICE__ -inline -float rhypotf(float x, float y) { return __ocml_rhypot_f32(x, y); } -__DEVICE__ -inline -float rintf(float x) { return __ocml_rint_f32(x); } -__DEVICE__ -inline -float rnorm3df(float x, float y, float z) -{ - return __ocml_rlen3_f32(x, y, z); -} - -__DEVICE__ -inline -float rnorm4df(float x, float y, float z, float w) -{ - return __ocml_rlen4_f32(x, y, z, w); -} -__DEVICE__ -inline -float rnormf(int dim, const float* a) -{ // TODO: placeholder until OCML adds support. - float r = 0; - while (dim--) { r += a[0] * a[0]; ++a; } - - return __ocml_rsqrt_f32(r); -} -__DEVICE__ -inline -float roundf(float x) { return __ocml_round_f32(x); } -__DEVICE__ -inline -float rsqrtf(float x) { return __ocml_rsqrt_f32(x); } -__DEVICE__ -inline -float scalblnf(float x, long int n) -{ - return (n < INT_MAX) ? __ocml_scalbn_f32(x, n) : __ocml_scalb_f32(x, n); -} -__DEVICE__ -inline -float scalbnf(float x, int n) { return __ocml_scalbn_f32(x, n); } -__DEVICE__ -inline -__RETURN_TYPE signbit(float x) { return __ocml_signbit_f32(x); } -__DEVICE__ -inline -void sincosf(float x, float* sptr, float* cptr) -{ - float tmp; - - *sptr = - __ocml_sincos_f32(x, (__attribute__((address_space(5))) float*) &tmp); - *cptr = tmp; -} -__DEVICE__ -inline -void sincospif(float x, float* sptr, float* cptr) -{ - float tmp; - - *sptr = - __ocml_sincospi_f32(x, (__attribute__((address_space(5))) float*) &tmp); - *cptr = tmp; -} -__DEVICE__ -inline -float sinf(float x) { return __ocml_sin_f32(x); } -__DEVICE__ -inline -float sinhf(float x) { return __ocml_sinh_f32(x); } -__DEVICE__ -inline -float sinpif(float x) { return __ocml_sinpi_f32(x); } -__DEVICE__ -inline -float sqrtf(float x) { return __ocml_sqrt_f32(x); } -__DEVICE__ -inline -float tanf(float x) { return __ocml_tan_f32(x); } -__DEVICE__ -inline -float tanhf(float x) { return __ocml_tanh_f32(x); } -__DEVICE__ -inline -float tgammaf(float x) { return __ocml_tgamma_f32(x); } -__DEVICE__ -inline -float truncf(float x) { return __ocml_trunc_f32(x); } -__DEVICE__ -inline -float y0f(float x) { return __ocml_y0_f32(x); } -__DEVICE__ -inline -float y1f(float x) { return __ocml_y1_f32(x); } -__DEVICE__ -inline -float ynf(int n, float x) -{ // TODO: we could use Ahmes multiplication and the Miller & Brown algorithm - // for linear recurrences to get O(log n) steps, but it's unclear if - // it'd be beneficial in this case. Placeholder until OCML adds - // support. - if (n == 0) return y0f(x); - if (n == 1) return y1f(x); - - float x0 = y0f(x); - float x1 = y1f(x); - for (int i = 1; i < n; ++i) { - float x2 = (2 * i) / x * x1 - x0; - x0 = x1; - x1 = x2; - } - - return x1; -} - -// BEGIN INTRINSICS -__DEVICE__ -inline -float __cosf(float x) { return __ocml_native_cos_f32(x); } -__DEVICE__ -inline -float __exp10f(float x) { return __ocml_native_exp10_f32(x); } -__DEVICE__ -inline -float __expf(float x) { return __ocml_native_exp_f32(x); } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fadd_rd(float x, float y) { return __ocml_add_rtn_f32(x, y); } -#endif -__DEVICE__ -inline -float __fadd_rn(float x, float y) { return x + y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fadd_ru(float x, float y) { return __ocml_add_rtp_f32(x, y); } -__DEVICE__ -inline -float __fadd_rz(float x, float y) { return __ocml_add_rtz_f32(x, y); } -__DEVICE__ -inline -float __fdiv_rd(float x, float y) { return __ocml_div_rtn_f32(x, y); } -#endif -__DEVICE__ -inline -float __fdiv_rn(float x, float y) { return x / y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fdiv_ru(float x, float y) { return __ocml_div_rtp_f32(x, y); } -__DEVICE__ -inline -float __fdiv_rz(float x, float y) { return __ocml_div_rtz_f32(x, y); } -#endif -__DEVICE__ -inline -float __fdividef(float x, float y) { return x / y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fmaf_rd(float x, float y, float z) -{ - return __ocml_fma_rtn_f32(x, y, z); -} -#endif -__DEVICE__ -inline -float __fmaf_rn(float x, float y, float z) -{ - return __ocml_fma_f32(x, y, z); -} -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fmaf_ru(float x, float y, float z) -{ - return __ocml_fma_rtp_f32(x, y, z); -} -__DEVICE__ -inline -float __fmaf_rz(float x, float y, float z) -{ - return __ocml_fma_rtz_f32(x, y, z); -} -__DEVICE__ -inline -float __fmul_rd(float x, float y) { return __ocml_mul_rtn_f32(x, y); } -#endif -__DEVICE__ -inline -float __fmul_rn(float x, float y) { return x * y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fmul_ru(float x, float y) { return __ocml_mul_rtp_f32(x, y); } -__DEVICE__ -inline -float __fmul_rz(float x, float y) { return __ocml_mul_rtz_f32(x, y); } -__DEVICE__ -inline -float __frcp_rd(float x) { return __llvm_amdgcn_rcp_f32(x); } -#endif -__DEVICE__ -inline -float __frcp_rn(float x) { return __llvm_amdgcn_rcp_f32(x); } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __frcp_ru(float x) { return __llvm_amdgcn_rcp_f32(x); } -__DEVICE__ -inline -float __frcp_rz(float x) { return __llvm_amdgcn_rcp_f32(x); } -#endif -__DEVICE__ -inline -float __frsqrt_rn(float x) { return __llvm_amdgcn_rsq_f32(x); } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fsqrt_rd(float x) { return __ocml_sqrt_rtn_f32(x); } -#endif -__DEVICE__ -inline -float __fsqrt_rn(float x) { return __ocml_native_sqrt_f32(x); } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fsqrt_ru(float x) { return __ocml_sqrt_rtp_f32(x); } -__DEVICE__ -inline -float __fsqrt_rz(float x) { return __ocml_sqrt_rtz_f32(x); } -__DEVICE__ -inline -float __fsub_rd(float x, float y) { return __ocml_sub_rtn_f32(x, y); } -#endif -__DEVICE__ -inline -float __fsub_rn(float x, float y) { return x - y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -float __fsub_ru(float x, float y) { return __ocml_sub_rtp_f32(x, y); } -__DEVICE__ -inline -float __fsub_rz(float x, float y) { return __ocml_sub_rtz_f32(x, y); } -#endif -__DEVICE__ -inline -float __log10f(float x) { return __ocml_native_log10_f32(x); } -__DEVICE__ -inline -float __log2f(float x) { return __ocml_native_log2_f32(x); } -__DEVICE__ -inline -float __logf(float x) { return __ocml_native_log_f32(x); } -__DEVICE__ -inline -float __powf(float x, float y) { return __ocml_pow_f32(x, y); } -__DEVICE__ -inline -float __saturatef(float x) { return (x < 0) ? 0 : ((x > 1) ? 1 : x); } -__DEVICE__ -inline -void __sincosf(float x, float* sptr, float* cptr) -{ - *sptr = __ocml_native_sin_f32(x); - *cptr = __ocml_native_cos_f32(x); -} -__DEVICE__ -inline -float __sinf(float x) { return __ocml_native_sin_f32(x); } -__DEVICE__ -inline -float __tanf(float x) { return __ocml_tan_f32(x); } -// END INTRINSICS -// END FLOAT - -// BEGIN DOUBLE -__DEVICE__ -inline -double abs(double x) { return __ocml_fabs_f64(x); } -__DEVICE__ -inline -double acos(double x) { return __ocml_acos_f64(x); } -__DEVICE__ -inline -double acosh(double x) { return __ocml_acosh_f64(x); } -__DEVICE__ -inline -double asin(double x) { return __ocml_asin_f64(x); } -__DEVICE__ -inline -double asinh(double x) { return __ocml_asinh_f64(x); } -__DEVICE__ -inline -double atan(double x) { return __ocml_atan_f64(x); } -__DEVICE__ -inline -double atan2(double x, double y) { return __ocml_atan2_f64(x, y); } -__DEVICE__ -inline -double atanh(double x) { return __ocml_atanh_f64(x); } -__DEVICE__ -inline -double cbrt(double x) { return __ocml_cbrt_f64(x); } -__DEVICE__ -inline -double ceil(double x) { return __ocml_ceil_f64(x); } -__DEVICE__ -inline -double copysign(double x, double y) { return __ocml_copysign_f64(x, y); } -__DEVICE__ -inline -double cos(double x) { return __ocml_cos_f64(x); } -__DEVICE__ -inline -double cosh(double x) { return __ocml_cosh_f64(x); } -__DEVICE__ -inline -double cospi(double x) { return __ocml_cospi_f64(x); } -__DEVICE__ -inline -double cyl_bessel_i0(double x) { return __ocml_i0_f64(x); } -__DEVICE__ -inline -double cyl_bessel_i1(double x) { return __ocml_i1_f64(x); } -__DEVICE__ -inline -double erf(double x) { return __ocml_erf_f64(x); } -__DEVICE__ -inline -double erfc(double x) { return __ocml_erfc_f64(x); } -__DEVICE__ -inline -double erfcinv(double x) { return __ocml_erfcinv_f64(x); } -__DEVICE__ -inline -double erfcx(double x) { return __ocml_erfcx_f64(x); } -__DEVICE__ -inline -double erfinv(double x) { return __ocml_erfinv_f64(x); } -__DEVICE__ -inline -double exp(double x) { return __ocml_exp_f64(x); } -__DEVICE__ -inline -double exp10(double x) { return __ocml_exp10_f64(x); } -__DEVICE__ -inline -double exp2(double x) { return __ocml_exp2_f64(x); } -__DEVICE__ -inline -double expm1(double x) { return __ocml_expm1_f64(x); } -__DEVICE__ -inline -double fabs(double x) { return __ocml_fabs_f64(x); } -__DEVICE__ -inline -double fdim(double x, double y) { return __ocml_fdim_f64(x, y); } -__DEVICE__ -inline -double floor(double x) { return __ocml_floor_f64(x); } -__DEVICE__ -inline -double fma(double x, double y, double z) { return __ocml_fma_f64(x, y, z); } -__DEVICE__ -inline -double fmax(double x, double y) { return __ocml_fmax_f64(x, y); } -__DEVICE__ -inline -double fmin(double x, double y) { return __ocml_fmin_f64(x, y); } -__DEVICE__ -inline -double fmod(double x, double y) { return __ocml_fmod_f64(x, y); } -__DEVICE__ -inline -double frexp(double x, int* nptr) -{ - int tmp; - double r = - __ocml_frexp_f64(x, (__attribute__((address_space(5))) int*) &tmp); - *nptr = tmp; - - return r; -} -__DEVICE__ -inline -double hypot(double x, double y) { return __ocml_hypot_f64(x, y); } -__DEVICE__ -inline -int ilogb(double x) { return __ocml_ilogb_f64(x); } -__DEVICE__ -inline -__RETURN_TYPE isfinite(double x) { return __ocml_isfinite_f64(x); } -__DEVICE__ -inline -__RETURN_TYPE isinf(double x) { return __ocml_isinf_f64(x); } -__DEVICE__ -inline -__RETURN_TYPE isnan(double x) { return __ocml_isnan_f64(x); } -__DEVICE__ -inline -double j0(double x) { return __ocml_j0_f64(x); } -__DEVICE__ -inline -double j1(double x) { return __ocml_j1_f64(x); } -__DEVICE__ -inline -double jn(int n, double x) -{ // TODO: we could use Ahmes multiplication and the Miller & Brown algorithm - // for linear recurrences to get O(log n) steps, but it's unclear if - // it'd be beneficial in this case. Placeholder until OCML adds - // support. - if (n == 0) return j0f(x); - if (n == 1) return j1f(x); - - double x0 = j0f(x); - double x1 = j1f(x); - for (int i = 1; i < n; ++i) { - double x2 = (2 * i) / x * x1 - x0; - x0 = x1; - x1 = x2; - } - - return x1; -} -__DEVICE__ -inline -double ldexp(double x, int e) { return __ocml_ldexp_f64(x, e); } -__DEVICE__ -inline -double lgamma(double x) { return __ocml_lgamma_f64(x); } -__DEVICE__ -inline -long long int llrint(double x) { return __ocml_rint_f64(x); } -__DEVICE__ -inline -long long int llround(double x) { return __ocml_round_f64(x); } -__DEVICE__ -inline -double log(double x) { return __ocml_log_f64(x); } -__DEVICE__ -inline -double log10(double x) { return __ocml_log10_f64(x); } -__DEVICE__ -inline -double log1p(double x) { return __ocml_log1p_f64(x); } -__DEVICE__ -inline -double log2(double x) { return __ocml_log2_f64(x); } -__DEVICE__ -inline -double logb(double x) { return __ocml_logb_f64(x); } -__DEVICE__ -inline -long int lrint(double x) { return __ocml_rint_f64(x); } -__DEVICE__ -inline -long int lround(double x) { return __ocml_round_f64(x); } -__DEVICE__ -inline -double modf(double x, double* iptr) -{ - double tmp; - double r = - __ocml_modf_f64(x, (__attribute__((address_space(5))) double*) &tmp); - *iptr = tmp; - - return r; -} -__DEVICE__ -inline -double nan(const char* tagp) -{ -#if !_WIN32 - union { - double val; - struct ieee_double { - uint64_t mantissa : 51; - uint32_t quiet : 1; - uint32_t exponent : 11; - uint32_t sign : 1; - } bits; - static_assert(sizeof(double) == sizeof(ieee_double), ""); - } tmp; - - tmp.bits.sign = 0u; - tmp.bits.exponent = ~0u; - tmp.bits.quiet = 1u; - tmp.bits.mantissa = __make_mantissa(tagp); - - return tmp.val; -#else - static_assert(sizeof(uint64_t)==sizeof(double)); - uint64_t val = __make_mantissa(tagp); - val |= 0xFFF << 51; - return *reinterpret_cast(&val); -#endif -} -__DEVICE__ -inline -double nearbyint(double x) { return __ocml_nearbyint_f64(x); } -__DEVICE__ -inline -double nextafter(double x, double y) { return __ocml_nextafter_f64(x, y); } -__DEVICE__ -inline -double norm(int dim, const double* a) -{ // TODO: placeholder until OCML adds support. - double r = 0; - while (dim--) { r += a[0] * a[0]; ++a; } - - return __ocml_sqrt_f64(r); -} -__DEVICE__ -inline -double norm3d(double x, double y, double z) -{ - return __ocml_len3_f64(x, y, z); -} -__DEVICE__ -inline -double norm4d(double x, double y, double z, double w) -{ - return __ocml_len4_f64(x, y, z, w); -} -__DEVICE__ -inline -double normcdf(double x) { return __ocml_ncdf_f64(x); } -__DEVICE__ -inline -double normcdfinv(double x) { return __ocml_ncdfinv_f64(x); } -__DEVICE__ -inline -double pow(double x, double y) { return __ocml_pow_f64(x, y); } -__DEVICE__ -inline -double powi(double base, int iexp) { return __ocml_pown_f64(base, iexp); } -__DEVICE__ -inline -double rcbrt(double x) { return __ocml_rcbrt_f64(x); } -__DEVICE__ -inline -double remainder(double x, double y) { return __ocml_remainder_f64(x, y); } -__DEVICE__ -inline -double remquo(double x, double y, int* quo) -{ - int tmp; - double r = - __ocml_remquo_f64(x, y, (__attribute__((address_space(5))) int*) &tmp); - *quo = tmp; - - return r; -} -__DEVICE__ -inline -double rhypot(double x, double y) { return __ocml_rhypot_f64(x, y); } -__DEVICE__ -inline -double rint(double x) { return __ocml_rint_f64(x); } -__DEVICE__ -inline -double rnorm(int dim, const double* a) -{ // TODO: placeholder until OCML adds support. - double r = 0; - while (dim--) { r += a[0] * a[0]; ++a; } - - return __ocml_rsqrt_f64(r); -} -__DEVICE__ -inline -double rnorm3d(double x, double y, double z) -{ - return __ocml_rlen3_f64(x, y, z); -} -__DEVICE__ -inline -double rnorm4d(double x, double y, double z, double w) -{ - return __ocml_rlen4_f64(x, y, z, w); -} -__DEVICE__ -inline -double round(double x) { return __ocml_round_f64(x); } -__DEVICE__ -inline -double rsqrt(double x) { return __ocml_rsqrt_f64(x); } -__DEVICE__ -inline -double scalbln(double x, long int n) -{ - return (n < INT_MAX) ? __ocml_scalbn_f64(x, n) : __ocml_scalb_f64(x, n); -} -__DEVICE__ -inline -double scalbn(double x, int n) { return __ocml_scalbn_f64(x, n); } -__DEVICE__ -inline -__RETURN_TYPE signbit(double x) { return __ocml_signbit_f64(x); } -__DEVICE__ -inline -double sin(double x) { return __ocml_sin_f64(x); } -__DEVICE__ -inline -void sincos(double x, double* sptr, double* cptr) -{ - double tmp; - *sptr = - __ocml_sincos_f64(x, (__attribute__((address_space(5))) double*) &tmp); - *cptr = tmp; -} -__DEVICE__ -inline -void sincospi(double x, double* sptr, double* cptr) -{ - double tmp; - *sptr = __ocml_sincospi_f64( - x, (__attribute__((address_space(5))) double*) &tmp); - *cptr = tmp; -} -__DEVICE__ -inline -double sinh(double x) { return __ocml_sinh_f64(x); } -__DEVICE__ -inline -double sinpi(double x) { return __ocml_sinpi_f64(x); } -__DEVICE__ -inline -double sqrt(double x) { return __ocml_sqrt_f64(x); } -__DEVICE__ -inline -double tan(double x) { return __ocml_tan_f64(x); } -__DEVICE__ -inline -double tanh(double x) { return __ocml_tanh_f64(x); } -__DEVICE__ -inline -double tgamma(double x) { return __ocml_tgamma_f64(x); } -__DEVICE__ -inline -double trunc(double x) { return __ocml_trunc_f64(x); } -__DEVICE__ -inline -double y0(double x) { return __ocml_y0_f64(x); } -__DEVICE__ -inline -double y1(double x) { return __ocml_y1_f64(x); } -__DEVICE__ -inline -double yn(int n, double x) -{ // TODO: we could use Ahmes multiplication and the Miller & Brown algorithm - // for linear recurrences to get O(log n) steps, but it's unclear if - // it'd be beneficial in this case. Placeholder until OCML adds - // support. - if (n == 0) return j0f(x); - if (n == 1) return j1f(x); - - double x0 = j0f(x); - double x1 = j1f(x); - for (int i = 1; i < n; ++i) { - double x2 = (2 * i) / x * x1 - x0; - x0 = x1; - x1 = x2; - } - - return x1; -} - -// BEGIN INTRINSICS -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -double __dadd_rd(double x, double y) { return __ocml_add_rtn_f64(x, y); } -#endif -__DEVICE__ -inline -double __dadd_rn(double x, double y) { return x + y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -double __dadd_ru(double x, double y) { return __ocml_add_rtp_f64(x, y); } -__DEVICE__ -inline -double __dadd_rz(double x, double y) { return __ocml_add_rtz_f64(x, y); } -__DEVICE__ -inline -double __ddiv_rd(double x, double y) { return __ocml_div_rtn_f64(x, y); } -#endif -__DEVICE__ -inline -double __ddiv_rn(double x, double y) { return x / y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -double __ddiv_ru(double x, double y) { return __ocml_div_rtp_f64(x, y); } -__DEVICE__ -inline -double __ddiv_rz(double x, double y) { return __ocml_div_rtz_f64(x, y); } -__DEVICE__ -inline -double __dmul_rd(double x, double y) { return __ocml_mul_rtn_f64(x, y); } -#endif -__DEVICE__ -inline -double __dmul_rn(double x, double y) { return x * y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -double __dmul_ru(double x, double y) { return __ocml_mul_rtp_f64(x, y); } -__DEVICE__ -inline -double __dmul_rz(double x, double y) { return __ocml_mul_rtz_f64(x, y); } -__DEVICE__ -inline -double __drcp_rd(double x) { return __llvm_amdgcn_rcp_f64(x); } -#endif -__DEVICE__ -inline -double __drcp_rn(double x) { return __llvm_amdgcn_rcp_f64(x); } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -double __drcp_ru(double x) { return __llvm_amdgcn_rcp_f64(x); } -__DEVICE__ -inline -double __drcp_rz(double x) { return __llvm_amdgcn_rcp_f64(x); } -__DEVICE__ -inline -double __dsqrt_rd(double x) { return __ocml_sqrt_rtn_f64(x); } -#endif -__DEVICE__ -inline -double __dsqrt_rn(double x) { return __ocml_sqrt_f64(x); } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -double __dsqrt_ru(double x) { return __ocml_sqrt_rtp_f64(x); } -__DEVICE__ -inline -double __dsqrt_rz(double x) { return __ocml_sqrt_rtz_f64(x); } -__DEVICE__ -inline -double __dsub_rd(double x, double y) { return __ocml_sub_rtn_f64(x, y); } -#endif -__DEVICE__ -inline -double __dsub_rn(double x, double y) { return x - y; } -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -double __dsub_ru(double x, double y) { return __ocml_sub_rtp_f64(x, y); } -__DEVICE__ -inline -double __dsub_rz(double x, double y) { return __ocml_sub_rtz_f64(x, y); } -__DEVICE__ -inline -double __fma_rd(double x, double y, double z) -{ - return __ocml_fma_rtn_f64(x, y, z); -} -#endif -__DEVICE__ -inline -double __fma_rn(double x, double y, double z) -{ - return __ocml_fma_f64(x, y, z); -} -#if defined OCML_BASIC_ROUNDED_OPERATIONS -__DEVICE__ -inline -double __fma_ru(double x, double y, double z) -{ - return __ocml_fma_rtp_f64(x, y, z); -} -__DEVICE__ -inline -double __fma_rz(double x, double y, double z) -{ - return __ocml_fma_rtz_f64(x, y, z); -} -#endif -// END INTRINSICS -// END DOUBLE - -// BEGIN INTEGER -__DEVICE__ -inline -int abs(int x) -{ - int sgn = x >> (sizeof(int) * CHAR_BIT - 1); - return (x ^ sgn) - sgn; -} -__DEVICE__ -inline -long labs(long x) -{ - long sgn = x >> (sizeof(long) * CHAR_BIT - 1); - return (x ^ sgn) - sgn; -} -__DEVICE__ -inline -long long llabs(long long x) -{ - long long sgn = x >> (sizeof(long long) * CHAR_BIT - 1); - return (x ^ sgn) - sgn; -} - -#if defined(__cplusplus) - __DEVICE__ - inline - long abs(long x) { return labs(x); } - __DEVICE__ - inline - long long abs(long long x) { return llabs(x); } -#endif -// END INTEGER - -__DEVICE__ -inline _Float16 fma(_Float16 x, _Float16 y, _Float16 z) { - return __ocml_fma_f16(x, y, z); -} - -__DEVICE__ -inline float fma(float x, float y, float z) { - return fmaf(x, y, z); -} - -#pragma push_macro("__DEF_FLOAT_FUN") -#pragma push_macro("__DEF_FLOAT_FUN2") -#pragma push_macro("__DEF_FLOAT_FUN2I") -#pragma push_macro("__HIP_OVERLOAD") -#pragma push_macro("__HIP_OVERLOAD2") - -// __hip_enable_if::type is a type function which returns __T if __B is true. -template -struct __hip_enable_if {}; - -template struct __hip_enable_if { - typedef __T type; -}; - -// __HIP_OVERLOAD1 is used to resolve function calls with integer argument to -// avoid compilation error due to ambibuity. e.g. floor(5) is resolved with -// floor(double). -#define __HIP_OVERLOAD1(__retty, __fn) \ - template \ - __DEVICE__ \ - typename __hip_enable_if::is_integer, \ - __retty>::type \ - __fn(__T __x) { \ - return ::__fn((double)__x); \ - } - -// __HIP_OVERLOAD2 is used to resolve function calls with mixed float/double -// or integer argument to avoid compilation error due to ambibuity. e.g. -// max(5.0f, 6.0) is resolved with max(double, double). -#define __HIP_OVERLOAD2(__retty, __fn) \ - template \ - __DEVICE__ typename __hip_enable_if< \ - std::numeric_limits<__T1>::is_specialized && \ - std::numeric_limits<__T2>::is_specialized, \ - __retty>::type \ - __fn(__T1 __x, __T2 __y) { \ - return __fn((double)__x, (double)__y); \ - } - -// Define cmath functions with float argument and returns float. -#define __DEF_FUN1(retty, func) \ -__DEVICE__ \ -inline \ -float func(float x) \ -{ \ - return func##f(x); \ -} \ -__HIP_OVERLOAD1(retty, func) - -// Define cmath functions with float argument and returns retty. -#define __DEF_FUNI(retty, func) \ -__DEVICE__ \ -inline \ -retty func(float x) \ -{ \ - return func##f(x); \ -} \ -__HIP_OVERLOAD1(retty, func) - -// define cmath functions with two float arguments. -#define __DEF_FUN2(retty, func) \ -__DEVICE__ \ -inline \ -float func(float x, float y) \ -{ \ - return func##f(x, y); \ -} \ -__HIP_OVERLOAD2(retty, func) - -__DEF_FUN1(double, acos) -__DEF_FUN1(double, acosh) -__DEF_FUN1(double, asin) -__DEF_FUN1(double, asinh) -__DEF_FUN1(double, atan) -__DEF_FUN2(double, atan2); -__DEF_FUN1(double, atanh) -__DEF_FUN1(double, cbrt) -__DEF_FUN1(double, ceil) -__DEF_FUN2(double, copysign); -__DEF_FUN1(double, cos) -__DEF_FUN1(double, cosh) -__DEF_FUN1(double, erf) -__DEF_FUN1(double, erfc) -__DEF_FUN1(double, exp) -__DEF_FUN1(double, exp2) -__DEF_FUN1(double, expm1) -__DEF_FUN1(double, fabs) -__DEF_FUN2(double, fdim); -__DEF_FUN1(double, floor) -__DEF_FUN2(double, fmax); -__DEF_FUN2(double, fmin); -__DEF_FUN2(double, fmod); -//__HIP_OVERLOAD1(int, fpclassify) -__DEF_FUN2(double, hypot); -__DEF_FUNI(int, ilogb) -__HIP_OVERLOAD1(bool, isfinite) -__HIP_OVERLOAD2(bool, isgreater); -__HIP_OVERLOAD2(bool, isgreaterequal); -__HIP_OVERLOAD1(bool, isinf); -__HIP_OVERLOAD2(bool, isless); -__HIP_OVERLOAD2(bool, islessequal); -__HIP_OVERLOAD2(bool, islessgreater); -__HIP_OVERLOAD1(bool, isnan); -//__HIP_OVERLOAD1(bool, isnormal) -__HIP_OVERLOAD2(bool, isunordered); -__DEF_FUN1(double, lgamma) -__DEF_FUN1(double, log) -__DEF_FUN1(double, log10) -__DEF_FUN1(double, log1p) -__DEF_FUN1(double, log2) -__DEF_FUN1(double, logb) -__DEF_FUNI(long long, llrint) -__DEF_FUNI(long long, llround) -__DEF_FUNI(long, lrint) -__DEF_FUNI(long, lround) -__DEF_FUN1(double, nearbyint); -__DEF_FUN2(double, nextafter); -__DEF_FUN2(double, pow); -__DEF_FUN2(double, remainder); -__DEF_FUN1(double, rint); -__DEF_FUN1(double, round); -__HIP_OVERLOAD1(bool, signbit) -__DEF_FUN1(double, sin) -__DEF_FUN1(double, sinh) -__DEF_FUN1(double, sqrt) -__DEF_FUN1(double, tan) -__DEF_FUN1(double, tanh) -__DEF_FUN1(double, tgamma) -__DEF_FUN1(double, trunc); - -// define cmath functions with a float and an integer argument. -#define __DEF_FLOAT_FUN2I(func) \ -__DEVICE__ \ -inline \ -float func(float x, int y) \ -{ \ - return func##f(x, y); \ -} -__DEF_FLOAT_FUN2I(scalbn) -__DEF_FLOAT_FUN2I(ldexp) - -template -__DEVICE__ inline T min(T arg1, T arg2) { - return (arg1 < arg2) ? arg1 : arg2; -} - -template -__DEVICE__ inline T max(T arg1, T arg2) { - return (arg1 > arg2) ? arg1 : arg2; -} - -__DEVICE__ inline int min(int arg1, int arg2) { - return (arg1 < arg2) ? arg1 : arg2; -} -__DEVICE__ inline int max(int arg1, int arg2) { - return (arg1 > arg2) ? arg1 : arg2; -} - -__DEVICE__ inline int min(uint32_t arg1, int arg2) { - return (arg1 < arg2) ? arg1 : arg2; -} -__DEVICE__ inline int max(uint32_t arg1, int arg2) { - return (arg1 > arg2) ? arg1 : arg2; -} - -__DEVICE__ -inline -float max(float x, float y) { - return fmaxf(x, y); -} - -__DEVICE__ -inline -double max(double x, double y) { - return fmax(x, y); -} - -__DEVICE__ -inline -float min(float x, float y) { - return fminf(x, y); -} - -__DEVICE__ -inline -double min(double x, double y) { - return fmin(x, y); -} - -__HIP_OVERLOAD2(double, max) -__HIP_OVERLOAD2(double, min) - -#if !defined(__HIPCC_RTC__) -__host__ inline static int min(int arg1, int arg2) { - return std::min(arg1, arg2); -} - -__host__ inline static int max(int arg1, int arg2) { - return std::max(arg1, arg2); -} -#endif // !defined(__HIPCC_RTC__) - -__DEVICE__ -inline float pow(float base, int iexp) { - return powif(base, iexp); -} - -__DEVICE__ -inline double pow(double base, int iexp) { - return powi(base, iexp); -} - -__DEVICE__ -inline _Float16 pow(_Float16 base, int iexp) { - return __ocml_pown_f16(base, iexp); -} - -#pragma pop_macro("__DEF_FLOAT_FUN") -#pragma pop_macro("__DEF_FLOAT_FUN2") -#pragma pop_macro("__DEF_FLOAT_FUN2I") -#pragma pop_macro("__HIP_OVERLOAD") -#pragma pop_macro("__HIP_OVERLOAD2") - -#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ - -#pragma pop_macro("__DEVICE__") -#pragma pop_macro("__RETURN_TYPE") - -// For backward compatibility. -// There are HIP applications e.g. TensorFlow, expecting __HIP_ARCH_* macros -// defined after including math_functions.h. -#include diff --git a/src/hipamd/include/hip/amd_detail/amd_surface_functions.h b/src/hipamd/include/hip/amd_detail/amd_surface_functions.h deleted file mode 100644 index 9e595b6e1a..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_surface_functions.h +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright (c) 2018 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_SURFACE_FUNCTIONS_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_SURFACE_FUNCTIONS_H - -#include - -#define __SURFACE_FUNCTIONS_DECL__ static inline __device__ -template -__SURFACE_FUNCTIONS_DECL__ void surf2Dread(T* data, hipSurfaceObject_t surfObj, int x, int y, - int boundaryMode = hipBoundaryModeZero) { - hipArray* arrayPtr = (hipArray*)surfObj; - size_t width = arrayPtr->width; - size_t height = arrayPtr->height; - int32_t xOffset = x / sizeof(T); - T* dataPtr = (T*)arrayPtr->data; - if ((xOffset > width) || (xOffset < 0) || (y > height) || (y < 0)) { - if (boundaryMode == hipBoundaryModeZero) { - *data = 0; - } - } else { - *data = *(dataPtr + y * width + xOffset); - } -} - -template -__SURFACE_FUNCTIONS_DECL__ void surf2Dwrite(T data, hipSurfaceObject_t surfObj, int x, int y, - int boundaryMode = hipBoundaryModeZero) { - hipArray* arrayPtr = (hipArray*)surfObj; - size_t width = arrayPtr->width; - size_t height = arrayPtr->height; - int32_t xOffset = x / sizeof(T); - T* dataPtr = (T*)arrayPtr->data; - if (!((xOffset > width) || (xOffset < 0) || (y > height) || (y < 0))) { - *(dataPtr + y * width + xOffset) = data; - } -} - -#endif diff --git a/src/hipamd/include/hip/amd_detail/amd_texture_types.h b/src/hipamd/include/hip/amd_detail/amd_texture_types.h deleted file mode 100644 index f9f0290e5b..0000000000 --- a/src/hipamd/include/hip/amd_detail/amd_texture_types.h +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_TEXTURE_TYPES_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_TEXTURE_TYPES_H - -#include - -#define hipTextureType1D 0x01 -#define hipTextureType2D 0x02 -#define hipTextureType3D 0x03 -#define hipTextureTypeCubemap 0x0C -#define hipTextureType1DLayered 0xF1 -#define hipTextureType2DLayered 0xF2 -#define hipTextureTypeCubemapLayered 0xFC - -/** - * Should be same as HSA_IMAGE_OBJECT_SIZE_DWORD/HSA_SAMPLER_OBJECT_SIZE_DWORD - */ -#define HIP_IMAGE_OBJECT_SIZE_DWORD 12 -#define HIP_SAMPLER_OBJECT_SIZE_DWORD 8 -#define HIP_SAMPLER_OBJECT_OFFSET_DWORD HIP_IMAGE_OBJECT_SIZE_DWORD -#define HIP_TEXTURE_OBJECT_SIZE_DWORD (HIP_IMAGE_OBJECT_SIZE_DWORD + HIP_SAMPLER_OBJECT_SIZE_DWORD) - -/** - * An opaque value that represents a hip texture object - */ -struct __hip_texture; -typedef struct __hip_texture* hipTextureObject_t; - -/** - * hip texture address modes - */ -enum hipTextureAddressMode { - hipAddressModeWrap = 0, - hipAddressModeClamp = 1, - hipAddressModeMirror = 2, - hipAddressModeBorder = 3 -}; - -/** - * hip texture filter modes - */ -enum hipTextureFilterMode { hipFilterModePoint = 0, hipFilterModeLinear = 1 }; - -/** - * hip texture read modes - */ -enum hipTextureReadMode { hipReadModeElementType = 0, hipReadModeNormalizedFloat = 1 }; - -/** - * hip texture reference - */ -typedef struct textureReference { - int normalized; - enum hipTextureReadMode readMode;// used only for driver API's - enum hipTextureFilterMode filterMode; - enum hipTextureAddressMode addressMode[3]; // Texture address mode for up to 3 dimensions - struct hipChannelFormatDesc channelDesc; - int sRGB; // Perform sRGB->linear conversion during texture read - unsigned int maxAnisotropy; // Limit to the anisotropy ratio - enum hipTextureFilterMode mipmapFilterMode; - float mipmapLevelBias; - float minMipmapLevelClamp; - float maxMipmapLevelClamp; - - hipTextureObject_t textureObject; - int numChannels; - enum hipArray_Format format; -}textureReference; - -/** - * hip texture descriptor - */ -typedef struct hipTextureDesc { - enum hipTextureAddressMode addressMode[3]; // Texture address mode for up to 3 dimensions - enum hipTextureFilterMode filterMode; - enum hipTextureReadMode readMode; - int sRGB; // Perform sRGB->linear conversion during texture read - float borderColor[4]; - int normalizedCoords; - unsigned int maxAnisotropy; - enum hipTextureFilterMode mipmapFilterMode; - float mipmapLevelBias; - float minMipmapLevelClamp; - float maxMipmapLevelClamp; -}hipTextureDesc; - -#endif diff --git a/src/hipamd/include/hip/amd_detail/concepts.hpp b/src/hipamd/include/hip/amd_detail/concepts.hpp deleted file mode 100644 index 373cefb292..0000000000 --- a/src/hipamd/include/hip/amd_detail/concepts.hpp +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright (c) 2015-present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -namespace hip_impl // Documentation only. -{ -#define requires(...) - -#define FunctionalProcedure typename -} // namespace hip_impl diff --git a/src/hipamd/include/hip/amd_detail/device_library_decls.h b/src/hipamd/include/hip/amd_detail/device_library_decls.h deleted file mode 100644 index bfb922ff8c..0000000000 --- a/src/hipamd/include/hip/amd_detail/device_library_decls.h +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/device_library_decls.h - * @brief Contains declarations for types and functions in device library. - * Uses int64_t and uint64_t instead of long, long long, unsigned - * long and unsigned long long types for device library API - * declarations. - */ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H - -#include "hip/amd_detail/host_defines.h" - -typedef unsigned char uchar; -typedef unsigned short ushort; -typedef unsigned int uint; -typedef unsigned long ulong; -typedef unsigned long long ullong; - -extern "C" __device__ __attribute__((const)) bool __ockl_wfany_i32(int); -extern "C" __device__ __attribute__((const)) bool __ockl_wfall_i32(int); -extern "C" __device__ uint __ockl_activelane_u32(void); - -extern "C" __device__ __attribute__((const)) uint __ockl_mul24_u32(uint, uint); -extern "C" __device__ __attribute__((const)) int __ockl_mul24_i32(int, int); -extern "C" __device__ __attribute__((const)) uint __ockl_mul_hi_u32(uint, uint); -extern "C" __device__ __attribute__((const)) int __ockl_mul_hi_i32(int, int); -extern "C" __device__ __attribute__((const)) uint __ockl_sadd_u32(uint, uint, uint); - -extern "C" __device__ __attribute__((const)) uchar __ockl_clz_u8(uchar); -extern "C" __device__ __attribute__((const)) ushort __ockl_clz_u16(ushort); -extern "C" __device__ __attribute__((const)) uint __ockl_clz_u32(uint); -extern "C" __device__ __attribute__((const)) uint64_t __ockl_clz_u64(uint64_t); - -extern "C" __device__ __attribute__((const)) float __ocml_floor_f32(float); -extern "C" __device__ __attribute__((const)) float __ocml_rint_f32(float); -extern "C" __device__ __attribute__((const)) float __ocml_ceil_f32(float); -extern "C" __device__ __attribute__((const)) float __ocml_trunc_f32(float); - -extern "C" __device__ __attribute__((const)) float __ocml_fmin_f32(float, float); -extern "C" __device__ __attribute__((const)) float __ocml_fmax_f32(float, float); - -extern "C" __device__ __attribute__((convergent)) void __ockl_gws_init(uint nwm1, uint rid); -extern "C" __device__ __attribute__((convergent)) void __ockl_gws_barrier(uint nwm1, uint rid); - -extern "C" __device__ __attribute__((const)) uint32_t __ockl_lane_u32(); -extern "C" __device__ __attribute__((const)) int __ockl_grid_is_valid(void); -extern "C" __device__ __attribute__((convergent)) void __ockl_grid_sync(void); -extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_num_grids(void); -extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_grid_rank(void); -extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_size(void); -extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_thread_rank(void); -extern "C" __device__ __attribute__((const)) int __ockl_multi_grid_is_valid(void); -extern "C" __device__ __attribute__((convergent)) void __ockl_multi_grid_sync(void); - -extern "C" __device__ void __ockl_atomic_add_noret_f32(float*, float); - -extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_add_i32(int a); -extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_and_i32(int a); -extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_or_i32(int a); - - -// Introduce local address space -#define __local __attribute__((address_space(3))) - -#ifdef __HIP_DEVICE_COMPILE__ -__device__ inline static __local void* __to_local(unsigned x) { return (__local void*)x; } -#endif //__HIP_DEVICE_COMPILE__ - -// Using hip.amdgcn.bc - sync threads -#define __CLK_LOCAL_MEM_FENCE 0x01 -typedef unsigned __cl_mem_fence_flags; - -typedef enum __memory_scope { - __memory_scope_work_item = __OPENCL_MEMORY_SCOPE_WORK_ITEM, - __memory_scope_work_group = __OPENCL_MEMORY_SCOPE_WORK_GROUP, - __memory_scope_device = __OPENCL_MEMORY_SCOPE_DEVICE, - __memory_scope_all_svm_devices = __OPENCL_MEMORY_SCOPE_ALL_SVM_DEVICES, - __memory_scope_sub_group = __OPENCL_MEMORY_SCOPE_SUB_GROUP -} __memory_scope; - -// enum values aligned with what clang uses in EmitAtomicExpr() -typedef enum __memory_order -{ - __memory_order_relaxed = __ATOMIC_RELAXED, - __memory_order_acquire = __ATOMIC_ACQUIRE, - __memory_order_release = __ATOMIC_RELEASE, - __memory_order_acq_rel = __ATOMIC_ACQ_REL, - __memory_order_seq_cst = __ATOMIC_SEQ_CST -} __memory_order; - -// Linked from hip.amdgcn.bc -extern "C" __device__ void -__atomic_work_item_fence(__cl_mem_fence_flags, __memory_order, __memory_scope); - -#endif diff --git a/src/hipamd/include/hip/amd_detail/functional_grid_launch.hpp b/src/hipamd/include/hip/amd_detail/functional_grid_launch.hpp deleted file mode 100644 index efe6a60197..0000000000 --- a/src/hipamd/include/hip/amd_detail/functional_grid_launch.hpp +++ /dev/null @@ -1,218 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#include "concepts.hpp" -#include "helpers.hpp" -#include "program_state.hpp" -#include "hip_runtime_api.h" - -#include -#include -#include -#include -#include -#include - -hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, int numDevices, - unsigned int flags, hip_impl::program_state& ps); - -hipError_t hipLaunchCooperativeKernel(const void* f, dim3 gridDim, - dim3 blockDim, void** args, - size_t sharedMem, hipStream_t stream, - hip_impl::program_state& ps); - -hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList, - int numDevices, - unsigned int flags, - hip_impl::program_state& ps); - -#pragma GCC visibility push(hidden) - -namespace hip_impl { -template {}>::type* = nullptr> -inline T round_up_to_next_multiple_nonnegative(T x, T y) { - T tmp = x + y - 1; - return tmp - tmp % y; -} - -template < - std::size_t n, - typename... Ts, - typename std::enable_if::type* = nullptr> -inline hip_impl::kernarg make_kernarg( - const std::tuple&, - const kernargs_size_align&, - hip_impl::kernarg kernarg) { - return kernarg; -} - -template < - std::size_t n, - typename... Ts, - typename std::enable_if::type* = nullptr> -inline hip_impl::kernarg make_kernarg( - const std::tuple& formals, - const kernargs_size_align& size_align, - hip_impl::kernarg kernarg) { - using T = typename std::tuple_element>::type; - - static_assert( - !std::is_reference{}, - "A __global__ function cannot have a reference as one of its " - "arguments."); - #if defined(HIP_STRICT) - static_assert( - std::is_trivially_copyable{}, - "Only TriviallyCopyable types can be arguments to a __global__ " - "function"); - #endif - - kernarg.resize(round_up_to_next_multiple_nonnegative( - kernarg.size(), size_align.alignment(n)) + size_align.size(n)); - - std::memcpy( - kernarg.data() + kernarg.size() - size_align.size(n), - &std::get(formals), - size_align.size(n)); - return make_kernarg(formals, size_align, std::move(kernarg)); -} - -template -inline hip_impl::kernarg make_kernarg( - void (*kernel)(Formals...), std::tuple actuals) { - static_assert(sizeof...(Formals) == sizeof...(Actuals), - "The count of formal arguments must match the count of actuals."); - - if (sizeof...(Formals) == 0) return {}; - - std::tuple to_formals{std::move(actuals)}; - hip_impl::kernarg kernarg; - kernarg.reserve(sizeof(to_formals)); - - auto& ps = hip_impl::get_program_state(); - return make_kernarg<0>(to_formals, - ps.get_kernargs_size_align( - reinterpret_cast(kernel)), - std::move(kernarg)); -} - - -HIP_INTERNAL_EXPORTED_API hsa_agent_t target_agent(hipStream_t stream); - -inline -__attribute__((visibility("hidden"))) -void hipLaunchKernelGGLImpl( - std::uintptr_t function_address, - const dim3& numBlocks, - const dim3& dimBlocks, - std::uint32_t sharedMemBytes, - hipStream_t stream, - void** kernarg) { - - const auto& kd = hip_impl::get_program_state().kernel_descriptor(function_address, - target_agent(stream)); - - hipModuleLaunchKernel(kd, numBlocks.x, numBlocks.y, numBlocks.z, - dimBlocks.x, dimBlocks.y, dimBlocks.z, sharedMemBytes, - stream, nullptr, kernarg); -} -} // Namespace hip_impl. - - -template -inline -hipError_t hipOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize, - T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0) { - - using namespace hip_impl; - - hip_impl::hip_init(); - auto f = get_program_state().kernel_descriptor(reinterpret_cast(kernel), - target_agent(0)); - - return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f, - dynSharedMemPerBlk, blockSizeLimit); -} - -template -inline -hipError_t hipOccupancyMaxPotentialBlockSizeWithFlags(int* gridSize, int* blockSize, - T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0, unsigned int flags = 0 ) { - - using namespace hip_impl; - - hip_impl::hip_init(); - if(flags != hipOccupancyDefault) return hipErrorNotSupported; - auto f = get_program_state().kernel_descriptor(reinterpret_cast(kernel), - target_agent(0)); - - return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f, - dynSharedMemPerBlk, blockSizeLimit); -} - -template -inline -void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks, - std::uint32_t sharedMemBytes, hipStream_t stream, - Args... args) { - hip_impl::hip_init(); - auto kernarg = hip_impl::make_kernarg(kernel, std::tuple{std::move(args)...}); - std::size_t kernarg_size = kernarg.size(); - - void* config[]{ - HIP_LAUNCH_PARAM_BUFFER_POINTER, - kernarg.data(), - HIP_LAUNCH_PARAM_BUFFER_SIZE, - &kernarg_size, - HIP_LAUNCH_PARAM_END}; - - hip_impl::hipLaunchKernelGGLImpl(reinterpret_cast(kernel), - numBlocks, dimBlocks, sharedMemBytes, - stream, &config[0]); -} - -template -inline -__attribute__((visibility("hidden"))) -hipError_t hipLaunchCooperativeKernel(F f, dim3 gridDim, dim3 blockDim, - void** args, size_t sharedMem, - hipStream_t stream) { - hip_impl::hip_init(); - auto& ps = hip_impl::get_program_state(); - return hipLaunchCooperativeKernel(reinterpret_cast(f), gridDim, - blockDim, args, sharedMem, stream, ps); -} - -inline -__attribute__((visibility("hidden"))) -hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList, - int numDevices, - unsigned int flags) { - - hip_impl::hip_init(); - auto& ps = hip_impl::get_program_state(); - return hipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags, ps); -} - -#pragma GCC visibility pop diff --git a/src/hipamd/include/hip/amd_detail/grid_launch.h b/src/hipamd/include/hip/amd_detail/grid_launch.h deleted file mode 100644 index 22841a5657..0000000000 --- a/src/hipamd/include/hip/amd_detail/grid_launch.h +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once - -#include - -#include - -#define GRID_LAUNCH_VERSION 20 - -// Extern definitions -namespace hc{ -class completion_future; -class accelerator_view; -} - - -// 3 dim structure for groups and grids. -typedef struct gl_dim3 -{ - int x,y,z; - gl_dim3(uint32_t _x=1, uint32_t _y=1, uint32_t _z=1) : x(_x), y(_y), z(_z) {}; -} gl_dim3; - -typedef enum gl_barrier_bit { - barrier_bit_queue_default, - barrier_bit_none, - barrier_bit_wait, -} gl_barrier_bit; - - -// grid_launch_parm contains information used to launch the kernel. -typedef struct grid_launch_parm -{ - //! Grid dimensions - gl_dim3 grid_dim; - - //! Group dimensions - gl_dim3 group_dim; - - //! Amount of dynamic group memory to use with the kernel launch. - //! This memory is in addition to the amount used statically in the kernel. - unsigned int dynamic_group_mem_bytes; - - //! Control setting of barrier bit on per-packet basis: - //! See gl_barrier_bit description. - //! Placeholder, is not used to control packet dispatch yet - enum gl_barrier_bit barrier_bit; - - //! Value of packet fences to apply to launch. - //! The correspond to the value of bits 9:14 in the AQL packet, - //! see HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE and hsa_fence_scope_t. - unsigned int launch_fence; - - //! Pointer to the accelerator_view where the kernel should execute. - //! If NULL, the default view on the default accelerator is used. - hc::accelerator_view *av; - - //! Pointer to the completion_future used to track the status of the command. - //! If NULL, the command does not write status. In this case, - //! synchronization can be enforced with queue-level waits or - //! waiting on younger commands. - hc::completion_future *cf; - - grid_launch_parm() = default; -} grid_launch_parm; - - -extern void init_grid_launch(grid_launch_parm *gl); diff --git a/src/hipamd/include/hip/amd_detail/grid_launch.hpp b/src/hipamd/include/hip/amd_detail/grid_launch.hpp deleted file mode 100644 index 04ce7e0366..0000000000 --- a/src/hipamd/include/hip/amd_detail/grid_launch.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include "grid_launch.h" -#include "hc.hpp" - -class grid_launch_parm_cxx : public grid_launch_parm -{ -public: - grid_launch_parm_cxx() = default; - - // customized serialization: don't need av and cf in kernel - __attribute__((annotate("serialize"))) - void __cxxamp_serialize(Kalmar::Serialize& s) const { - s.Append(sizeof(int), &grid_dim.x); - s.Append(sizeof(int), &grid_dim.y); - s.Append(sizeof(int), &grid_dim.z); - s.Append(sizeof(int), &group_dim.x); - s.Append(sizeof(int), &group_dim.y); - s.Append(sizeof(int), &group_dim.z); - } - - __attribute__((annotate("user_deserialize"))) - grid_launch_parm_cxx(int grid_dim_x, int grid_dim_y, int grid_dim_z, - int group_dim_x, int group_dim_y, int group_dim_z) { - grid_dim.x = grid_dim_x; - grid_dim.y = grid_dim_y; - grid_dim.z = grid_dim_z; - group_dim.x = group_dim_x; - group_dim.y = group_dim_y; - group_dim.z = group_dim_z; - } -}; - - -extern inline void grid_launch_init(grid_launch_parm *lp) { - lp->grid_dim.x = lp->grid_dim.y = lp->grid_dim.z = 1; - - lp->group_dim.x = lp->group_dim.y = lp->group_dim.z = 1; - - lp->dynamic_group_mem_bytes = 0; - - lp->barrier_bit = barrier_bit_queue_default; - lp->launch_fence = -1; - - // TODO - set to NULL? - static hc::accelerator_view av = hc::accelerator().get_default_view(); - lp->av = &av; - lp->cf = NULL; -} - diff --git a/src/hipamd/include/hip/amd_detail/grid_launch_GGL.hpp b/src/hipamd/include/hip/amd_detail/grid_launch_GGL.hpp deleted file mode 100644 index fbae198af1..0000000000 --- a/src/hipamd/include/hip/amd_detail/grid_launch_GGL.hpp +++ /dev/null @@ -1,26 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ -#pragma once - -#if GENERIC_GRID_LAUNCH == 1 -#include "macro_based_grid_launch.hpp" -#endif // GENERIC_GRID_LAUNCH \ No newline at end of file diff --git a/src/hipamd/include/hip/amd_detail/helpers.hpp b/src/hipamd/include/hip/amd_detail/helpers.hpp deleted file mode 100644 index b94b126994..0000000000 --- a/src/hipamd/include/hip/amd_detail/helpers.hpp +++ /dev/null @@ -1,137 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once -#include "concepts.hpp" - -#include // For std::conditional, std::decay, std::enable_if, - // std::false_type, std result_of and std::true_type. -#include // For std::declval. - -#ifdef __has_include // Check if __has_include is present -# if __has_include() // Check for version header -# include -# if defined(__cpp_lib_is_invocable) && !defined(HIP_HAS_INVOCABLE) -# define HIP_HAS_INVOCABLE __cpp_lib_is_invocable -# endif -# if defined(__cpp_lib_result_of_sfinae) && !defined(HIP_HAS_RESULT_OF_SFINAE) -# define HIP_HAS_RESULT_OF_SFINAE __cpp_lib_result_of_sfinae -# endif -# endif -#endif - -#ifndef HIP_HAS_INVOCABLE -#define HIP_HAS_INVOCABLE 0 -#endif - -#ifndef HIP_HAS_RESULT_OF_SFINAE -#define HIP_HAS_RESULT_OF_SFINAE 0 -#endif - -namespace std { // TODO: these should be removed as soon as possible. -#if (__cplusplus < 201406L) -#if (__cplusplus < 201402L) -template -using enable_if_t = typename enable_if::type; -template -using conditional_t = typename conditional::type; -template -using decay_t = typename decay::type; -template -using result_of_t = typename result_of::type; -template -using remove_reference_t = typename remove_reference::type; -#endif -#endif -} // namespace std - -namespace hip_impl { -template -using void_t_ = void; - -#if HIP_HAS_INVOCABLE -template -struct is_callable_impl; - -template -struct is_callable_impl : std::is_invocable {}; -#elif HIP_HAS_RESULT_OF_SFINAE -template -struct is_callable_impl : std::false_type {}; - -template -struct is_callable_impl::type > > : std::true_type {}; -#else -template -auto simple_invoke(T Base::*pmd, Derived&& ref) --> decltype(static_cast(ref).*pmd); - -template -auto simple_invoke(PMD&& pmd, Pointer&& ptr) --> decltype((*static_cast(ptr)).*static_cast(pmd)); - -template -auto simple_invoke(T Base::*pmd, const std::reference_wrapper& ref) --> decltype(ref.get().*pmd); - -template -auto simple_invoke(T Base::*pmf, Derived&& ref, Args&&... args) --> decltype((static_cast(ref).*pmf)(static_cast(args)...)); - -template -auto simple_invoke(PMF&& pmf, Pointer&& ptr, Args&&... args) --> decltype(((*static_cast(ptr)).*static_cast(pmf))(static_cast(args)...)); - -template -auto simple_invoke(T Base::*pmf, const std::reference_wrapper& ref, Args&&... args) --> decltype((ref.get().*pmf)(static_cast(args)...)); - -template -auto simple_invoke(F&& f, Ts&&... xs) --> decltype(f(static_cast(xs)...)); - -template -struct is_callable_impl : std::false_type {}; - -template -struct is_callable_impl(), std::declval()...))> > - : std::true_type {}; - -#endif - -template -struct is_callable : is_callable_impl {}; - -#define count_macro_args_impl_hip_(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \ - _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, \ - _26, _27, _28, _29, _30, _31, _n, ...) \ - _n -#define count_macro_args_hip_(...) \ - count_macro_args_impl_hip_(, ##__VA_ARGS__, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, \ - 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, \ - 0) - -#define overloaded_macro_expand_hip_(macro, arg_cnt) macro##arg_cnt -#define overload_macro_impl_hip_(macro, arg_cnt) overloaded_macro_expand_hip_(macro, arg_cnt) -#define overload_macro_hip_(macro, ...) \ - overload_macro_impl_hip_(macro, count_macro_args_hip_(__VA_ARGS__))(__VA_ARGS__) -} // namespace hip_impl diff --git a/src/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h b/src/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h deleted file mode 100644 index 6ef96e9a17..0000000000 --- a/src/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/hip_cooperative_groups_helper.h - * - * @brief Device side implementation of cooperative group feature. - * - * Defines helper constructs and APIs which aid the types and device API - * wrappers defined within `amd_detail/hip_cooperative_groups.h`. - */ -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H - -#if __cplusplus -#include - -#if !defined(__align__) -#define __align__(x) __attribute__((aligned(x))) -#endif - -#if !defined(__CG_QUALIFIER__) -#define __CG_QUALIFIER__ __device__ __forceinline__ -#endif - -#if !defined(__CG_STATIC_QUALIFIER__) -#define __CG_STATIC_QUALIFIER__ __device__ static __forceinline__ -#endif - -#if !defined(_CG_STATIC_CONST_DECL_) -#define _CG_STATIC_CONST_DECL_ static constexpr -#endif - -#if !defined(WAVEFRONT_SIZE) -#if __gfx1010__ || __gfx1011__ || __gfx1012__ || __gfx1030__ || __gfx1031__ -#define WAVEFRONT_SIZE 32 -#else -#define WAVEFRONT_SIZE 64 -#endif - -namespace cooperative_groups { - -/* Global scope */ -template -using is_power_of_2 = std::integral_constant; - -template -using is_valid_wavefront = std::integral_constant; - -template -using is_valid_tile_size = - std::integral_constant::value && is_valid_wavefront::value>; - -template -using is_valid_type = - std::integral_constant::value || std::is_floating_point::value>; - -namespace internal { - -/** \brief Enums representing different cooperative group types - */ -typedef enum { - cg_invalid, - cg_multi_grid, - cg_grid, - cg_workgroup, - cg_tiled_group -} group_type; - -/** - * Functionalities related to multi-grid cooperative group type - */ -namespace multi_grid { - -__CG_STATIC_QUALIFIER__ uint32_t num_grids() { return (uint32_t)__ockl_multi_grid_num_grids(); } - -__CG_STATIC_QUALIFIER__ uint32_t grid_rank() { return (uint32_t)__ockl_multi_grid_grid_rank(); } - -__CG_STATIC_QUALIFIER__ uint32_t size() { return (uint32_t)__ockl_multi_grid_size(); } - -__CG_STATIC_QUALIFIER__ uint32_t thread_rank() { return (uint32_t)__ockl_multi_grid_thread_rank(); } - -__CG_STATIC_QUALIFIER__ bool is_valid() { return (bool)__ockl_multi_grid_is_valid(); } - -__CG_STATIC_QUALIFIER__ void sync() { __ockl_multi_grid_sync(); } - -} // namespace multi_grid - -/** - * Functionalities related to grid cooperative group type - */ -namespace grid { - -__CG_STATIC_QUALIFIER__ uint32_t size() { - return (uint32_t)((hipBlockDim_z * hipGridDim_z) * (hipBlockDim_y * hipGridDim_y) * - (hipBlockDim_x * hipGridDim_x)); -} - -__CG_STATIC_QUALIFIER__ uint32_t thread_rank() { - // Compute global id of the workgroup to which the current thread belongs to - uint32_t blkIdx = (uint32_t)((hipBlockIdx_z * hipGridDim_y * hipGridDim_x) + - (hipBlockIdx_y * hipGridDim_x) + (hipBlockIdx_x)); - - // Compute total number of threads being passed to reach current workgroup - // within grid - uint32_t num_threads_till_current_workgroup = - (uint32_t)(blkIdx * (hipBlockDim_x * hipBlockDim_y * hipBlockDim_z)); - - // Compute thread local rank within current workgroup - uint32_t local_thread_rank = (uint32_t)((hipThreadIdx_z * hipBlockDim_y * hipBlockDim_x) + - (hipThreadIdx_y * hipBlockDim_x) + (hipThreadIdx_x)); - - return (num_threads_till_current_workgroup + local_thread_rank); -} - -__CG_STATIC_QUALIFIER__ bool is_valid() { return (bool)__ockl_grid_is_valid(); } - -__CG_STATIC_QUALIFIER__ void sync() { __ockl_grid_sync(); } - -} // namespace grid - -/** - * Functionalities related to `workgroup` (thread_block in CUDA terminology) - * cooperative group type - */ -namespace workgroup { - -__CG_STATIC_QUALIFIER__ dim3 group_index() { - return (dim3((uint32_t)hipBlockIdx_x, (uint32_t)hipBlockIdx_y, (uint32_t)hipBlockIdx_z)); -} - -__CG_STATIC_QUALIFIER__ dim3 thread_index() { - return (dim3((uint32_t)hipThreadIdx_x, (uint32_t)hipThreadIdx_y, (uint32_t)hipThreadIdx_z)); -} - -__CG_STATIC_QUALIFIER__ uint32_t size() { - return ((uint32_t)(hipBlockDim_x * hipBlockDim_y * hipBlockDim_z)); -} - -__CG_STATIC_QUALIFIER__ uint32_t thread_rank() { - return ((uint32_t)((hipThreadIdx_z * hipBlockDim_y * hipBlockDim_x) + - (hipThreadIdx_y * hipBlockDim_x) + (hipThreadIdx_x))); -} - -__CG_STATIC_QUALIFIER__ bool is_valid() { - // TODO(mahesha) any functionality need to be added here? I believe not - return true; -} - -__CG_STATIC_QUALIFIER__ void sync() { __syncthreads(); } - -} // namespace workgroup - -} // namespace internal - -} // namespace cooperative_groups - -#endif // __cplusplus -#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H -#endif diff --git a/src/hipamd/include/hip/amd_detail/hip_fp16_gcc.h b/src/hipamd/include/hip/amd_detail/hip_fp16_gcc.h deleted file mode 100644 index e76a7fff3a..0000000000 --- a/src/hipamd/include/hip/amd_detail/hip_fp16_gcc.h +++ /dev/null @@ -1,254 +0,0 @@ -#pragma once - -#if defined(__cplusplus) - #include -#endif - -struct __half_raw { - unsigned short x; -}; - -struct __half2_raw { - unsigned short x; - unsigned short y; -}; - -#if defined(__cplusplus) - struct __half; - - __half __float2half(float); - float __half2float(__half); - - // BEGIN STRUCT __HALF - struct __half { - protected: - unsigned short __x; - public: - // CREATORS - __half() = default; - __half(const __half_raw& x) : __x{x.x} {} - #if !defined(__HIP_NO_HALF_CONVERSIONS__) - __half(float x) : __x{__float2half(x).__x} {} - __half(double x) : __x{__float2half(x).__x} {} - #endif - __half(const __half&) = default; - __half(__half&&) = default; - ~__half() = default; - - // MANIPULATORS - __half& operator=(const __half&) = default; - __half& operator=(__half&&) = default; - __half& operator=(const __half_raw& x) { __x = x.x; return *this; } - #if !defined(__HIP_NO_HALF_CONVERSIONS__) - __half& operator=(float x) - { - __x = __float2half(x).__x; - return *this; - } - __half& operator=(double x) - { - return *this = static_cast(x); - } - #endif - - // ACCESSORS - operator float() const { return __half2float(*this); } - operator __half_raw() const { return __half_raw{__x}; } - }; - // END STRUCT __HALF - - // BEGIN STRUCT __HALF2 - struct __half2 { - public: - __half x; - __half y; - - // CREATORS - __half2() = default; - __half2(const __half2_raw& ix) - : - x{reinterpret_cast(ix.x)}, - y{reinterpret_cast(ix.y)} - {} - __half2(const __half& ix, const __half& iy) : x{ix}, y{iy} {} - __half2(const __half2&) = default; - __half2(__half2&&) = default; - ~__half2() = default; - - // MANIPULATORS - __half2& operator=(const __half2&) = default; - __half2& operator=(__half2&&) = default; - __half2& operator=(const __half2_raw& ix) - { - x = reinterpret_cast(ix.x); - y = reinterpret_cast(ix.y); - return *this; - } - - // ACCESSORS - operator __half2_raw() const - { - return __half2_raw{ - reinterpret_cast(x), - reinterpret_cast(y)}; - } - }; - // END STRUCT __HALF2 - - inline - unsigned short __internal_float2half( - float flt, unsigned int& sgn, unsigned int& rem) - { - unsigned int x{}; - std::memcpy(&x, &flt, sizeof(flt)); - - unsigned int u = (x & 0x7fffffffU); - sgn = ((x >> 16) & 0x8000U); - - // NaN/+Inf/-Inf - if (u >= 0x7f800000U) { - rem = 0; - return static_cast( - (u == 0x7f800000U) ? (sgn | 0x7c00U) : 0x7fffU); - } - // Overflows - if (u > 0x477fefffU) { - rem = 0x80000000U; - return static_cast(sgn | 0x7bffU); - } - // Normal numbers - if (u >= 0x38800000U) { - rem = u << 19; - u -= 0x38000000U; - return static_cast(sgn | (u >> 13)); - } - // +0/-0 - if (u < 0x33000001U) { - rem = u; - return static_cast(sgn); - } - // Denormal numbers - unsigned int exponent = u >> 23; - unsigned int mantissa = (u & 0x7fffffU); - unsigned int shift = 0x7eU - exponent; - mantissa |= 0x800000U; - rem = mantissa << (32 - shift); - return static_cast(sgn | (mantissa >> shift)); - } - - inline - __half __float2half(float x) - { - __half_raw r; - unsigned int sgn{}; - unsigned int rem{}; - r.x = __internal_float2half(x, sgn, rem); - if (rem > 0x80000000U || (rem == 0x80000000U && (r.x & 0x1))) ++r.x; - - return r; - } - - inline - __half __float2half_rn(float x) { return __float2half(x); } - - inline - __half __float2half_rz(float x) - { - __half_raw r; - unsigned int sgn{}; - unsigned int rem{}; - r.x = __internal_float2half(x, sgn, rem); - - return r; - } - - inline - __half __float2half_rd(float x) - { - __half_raw r; - unsigned int sgn{}; - unsigned int rem{}; - r.x = __internal_float2half(x, sgn, rem); - if (rem && sgn) ++r.x; - - return r; - } - - inline - __half __float2half_ru(float x) - { - __half_raw r; - unsigned int sgn{}; - unsigned int rem{}; - r.x = __internal_float2half(x, sgn, rem); - if (rem && !sgn) ++r.x; - - return r; - } - - inline - __half2 __float2half2_rn(float x) - { - return __half2{__float2half_rn(x), __float2half_rn(x)}; - } - - inline - __half2 __floats2half2_rn(float x, float y) - { - return __half2{__float2half_rn(x), __float2half_rn(y)}; - } - - inline - float __internal_half2float(unsigned short x) - { - unsigned int sign = ((x >> 15) & 1); - unsigned int exponent = ((x >> 10) & 0x1f); - unsigned int mantissa = ((x & 0x3ff) << 13); - - if (exponent == 0x1fU) { /* NaN or Inf */ - mantissa = (mantissa ? (sign = 0, 0x7fffffU) : 0); - exponent = 0xffU; - } else if (!exponent) { /* Denorm or Zero */ - if (mantissa) { - unsigned int msb; - exponent = 0x71U; - do { - msb = (mantissa & 0x400000U); - mantissa <<= 1; /* normalize */ - --exponent; - } while (!msb); - mantissa &= 0x7fffffU; /* 1.mantissa is implicit */ - } - } else { - exponent += 0x70U; - } - unsigned int u = ((sign << 31) | (exponent << 23) | mantissa); - float f; - memcpy(&f, &u, sizeof(u)); - - return f; - } - - inline - float __half2float(__half x) - { - return __internal_half2float(static_cast<__half_raw>(x).x); - } - - inline - float __low2float(__half2 x) - { - return __internal_half2float(static_cast<__half2_raw>(x).x); - } - - inline - float __high2float(__half2 x) - { - return __internal_half2float(static_cast<__half2_raw>(x).y); - } - - #if !defined(HIP_NO_HALF) - using half = __half; - using half2 = __half2; - #endif -#endif // defined(__cplusplus) diff --git a/src/hipamd/include/hip/amd_detail/hip_fp16_math_fwd.h b/src/hipamd/include/hip/amd_detail/hip_fp16_math_fwd.h deleted file mode 100644 index 7d2cf22bc3..0000000000 --- a/src/hipamd/include/hip/amd_detail/hip_fp16_math_fwd.h +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -// /* -// Half Math Functions -// */ - -#include "host_defines.h" -#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -extern "C" -{ - __device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16); - __device__ _Float16 __ocml_cos_f16(_Float16); - __device__ __attribute__((pure)) _Float16 __ocml_exp_f16(_Float16); - __device__ __attribute__((pure)) _Float16 __ocml_exp10_f16(_Float16); - __device__ __attribute__((pure)) _Float16 __ocml_exp2_f16(_Float16); - __device__ __attribute__((const)) _Float16 __ocml_floor_f16(_Float16); - __device__ __attribute__((const)) - _Float16 __ocml_fma_f16(_Float16, _Float16, _Float16); - __device__ __attribute__((const)) _Float16 __ocml_fabs_f16(_Float16); - __device__ __attribute__((const)) int __ocml_isinf_f16(_Float16); - __device__ __attribute__((const)) int __ocml_isnan_f16(_Float16); - __device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16); - __device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16); - __device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16); - __device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int); - __device__ __attribute__((const)) _Float16 __llvm_amdgcn_rcp_f16(_Float16); - __device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16); - __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16); - __device__ _Float16 __ocml_sin_f16(_Float16); - __device__ __attribute__((const)) _Float16 __ocml_sqrt_f16(_Float16); - __device__ __attribute__((const)) _Float16 __ocml_trunc_f16(_Float16); - - typedef _Float16 __2f16 __attribute__((ext_vector_type(2))); - typedef short __2i16 __attribute__((ext_vector_type(2))); - - #if __HIP_CLANG_ONLY__ - __device__ __attribute__((const)) float __ockl_fdot2(__2f16 a, __2f16 b, float c, bool s); - #endif - - __device__ __attribute__((const)) __2f16 __ocml_ceil_2f16(__2f16); - __device__ __attribute__((const)) __2f16 __ocml_fabs_2f16(__2f16); - __device__ __2f16 __ocml_cos_2f16(__2f16); - __device__ __attribute__((pure)) __2f16 __ocml_exp_2f16(__2f16); - __device__ __attribute__((pure)) __2f16 __ocml_exp10_2f16(__2f16); - __device__ __attribute__((pure)) __2f16 __ocml_exp2_2f16(__2f16); - __device__ __attribute__((const)) __2f16 __ocml_floor_2f16(__2f16); - __device__ __attribute__((const)) __2f16 __ocml_fma_2f16(__2f16, __2f16, __2f16); - __device__ __attribute__((const)) __2i16 __ocml_isinf_2f16(__2f16); - __device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16); - __device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16); - __device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16); - __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16); - __device__ inline - __2f16 __llvm_amdgcn_rcp_2f16(__2f16 x) // Not currently exposed by ROCDL. - { - return __2f16{__llvm_amdgcn_rcp_f16(x.x), __llvm_amdgcn_rcp_f16(x.y)}; - } - __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16); - __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16); - __device__ __2f16 __ocml_sin_2f16(__2f16); - __device__ __attribute__((const)) __2f16 __ocml_sqrt_2f16(__2f16); - __device__ __attribute__((const)) __2f16 __ocml_trunc_2f16(__2f16); -} -#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ diff --git a/src/hipamd/include/hip/amd_detail/hip_ldg.h b/src/hipamd/include/hip/amd_detail/hip_ldg.h deleted file mode 100644 index 91e76e6564..0000000000 --- a/src/hipamd/include/hip/amd_detail/hip_ldg.h +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_LDG_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_LDG_H - -#if __HIP_CLANG_ONLY__ -#include "amd_hip_vector_types.h" -#include "host_defines.h" - -__device__ inline static char __ldg(const char* ptr) { return *ptr; } - -__device__ inline static char2 __ldg(const char2* ptr) { return *ptr; } - -__device__ inline static char4 __ldg(const char4* ptr) { return *ptr; } - -__device__ inline static signed char __ldg(const signed char* ptr) { return ptr[0]; } - -__device__ inline static unsigned char __ldg(const unsigned char* ptr) { return ptr[0]; } - - -__device__ inline static short __ldg(const short* ptr) { return ptr[0]; } - -__device__ inline static short2 __ldg(const short2* ptr) { return ptr[0]; } - -__device__ inline static short4 __ldg(const short4* ptr) { return ptr[0]; } - -__device__ inline static unsigned short __ldg(const unsigned short* ptr) { return ptr[0]; } - - -__device__ inline static int __ldg(const int* ptr) { return ptr[0]; } - -__device__ inline static int2 __ldg(const int2* ptr) { return ptr[0]; } - -__device__ inline static int4 __ldg(const int4* ptr) { return ptr[0]; } - -__device__ inline static unsigned int __ldg(const unsigned int* ptr) { return ptr[0]; } - - -__device__ inline static long __ldg(const long* ptr) { return ptr[0]; } - -__device__ inline static unsigned long __ldg(const unsigned long* ptr) { return ptr[0]; } - - -__device__ inline static long long __ldg(const long long* ptr) { return ptr[0]; } - -__device__ inline static longlong2 __ldg(const longlong2* ptr) { return ptr[0]; } - -__device__ inline static unsigned long long __ldg(const unsigned long long* ptr) { return ptr[0]; } - - -__device__ inline static uchar2 __ldg(const uchar2* ptr) { return ptr[0]; } - -__device__ inline static uchar4 __ldg(const uchar4* ptr) { return ptr[0]; } - - -__device__ inline static ushort2 __ldg(const ushort2* ptr) { return ptr[0]; } - - -__device__ inline static uint2 __ldg(const uint2* ptr) { return ptr[0]; } - -__device__ inline static uint4 __ldg(const uint4* ptr) { return ptr[0]; } - - -__device__ inline static ulonglong2 __ldg(const ulonglong2* ptr) { return ptr[0]; } - - -__device__ inline static float __ldg(const float* ptr) { return ptr[0]; } - -__device__ inline static float2 __ldg(const float2* ptr) { return ptr[0]; } - -__device__ inline static float4 __ldg(const float4* ptr) { return ptr[0]; } - - -__device__ inline static double __ldg(const double* ptr) { return ptr[0]; } - -__device__ inline static double2 __ldg(const double2* ptr) { return ptr[0]; } - -#endif // __HIP_CLANG_ONLY__ - -#endif // HIP_LDG_H diff --git a/src/hipamd/include/hip/amd_detail/hip_memory.h b/src/hipamd/include/hip/amd_detail/hip_memory.h deleted file mode 100644 index f2c01633ea..0000000000 --- a/src/hipamd/include/hip/amd_detail/hip_memory.h +++ /dev/null @@ -1,114 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_MEMORY_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_MEMORY_H - -// Implementation of malloc and free device functions. -// HIP heap is implemented as a global array with fixed size. Users may define -// __HIP_SIZE_OF_PAGE and __HIP_NUM_PAGES to have a larger heap. - -#if __HIP__ && __HIP_ENABLE_DEVICE_MALLOC__ - -// Size of page in bytes. -#ifndef __HIP_SIZE_OF_PAGE -#define __HIP_SIZE_OF_PAGE 64 -#endif - -// Total number of pages -#ifndef __HIP_NUM_PAGES -#define __HIP_NUM_PAGES (16 * 64 * 64) -#endif - -#define __HIP_SIZE_OF_HEAP (__HIP_NUM_PAGES * __HIP_SIZE_OF_PAGE) - -#if __HIP_DEVICE_COMPILE__ -__attribute__((weak)) __device__ char __hip_device_heap[__HIP_SIZE_OF_HEAP]; -__attribute__((weak)) __device__ - uint32_t __hip_device_page_flag[__HIP_NUM_PAGES]; -#else -extern __device__ char __hip_device_heap[]; -extern __device__ uint32_t __hip_device_page_flag[]; -#endif - -extern "C" inline __device__ void* __hip_malloc(size_t size) { - char* heap = (char*)__hip_device_heap; - if (size > __HIP_SIZE_OF_HEAP) { - return (void*)nullptr; - } - uint32_t totalThreads = - hipBlockDim_x * hipGridDim_x * hipBlockDim_y - * hipGridDim_y * hipBlockDim_z * hipGridDim_z; - uint32_t currentWorkItem = hipThreadIdx_x + hipBlockDim_x * hipBlockIdx_x - + (hipThreadIdx_y + hipBlockDim_y * hipBlockIdx_y) * hipBlockDim_x - + (hipThreadIdx_z + hipBlockDim_z * hipBlockIdx_z) * hipBlockDim_x - * hipBlockDim_y; - - uint32_t numHeapsPerWorkItem = __HIP_NUM_PAGES / totalThreads; - uint32_t heapSizePerWorkItem = __HIP_SIZE_OF_HEAP / totalThreads; - - uint32_t stride = size / __HIP_SIZE_OF_PAGE; - uint32_t start = numHeapsPerWorkItem * currentWorkItem; - - uint32_t k = 0; - - while (__hip_device_page_flag[k] > 0) { - k++; - } - - for (uint32_t i = 0; i < stride - 1; i++) { - __hip_device_page_flag[i + start + k] = 1; - } - - __hip_device_page_flag[start + stride - 1 + k] = 2; - - void* ptr = (void*)(heap - + heapSizePerWorkItem * currentWorkItem + k * __HIP_SIZE_OF_PAGE); - - return ptr; -} - -extern "C" inline __device__ void* __hip_free(void* ptr) { - if (ptr == nullptr) { - return nullptr; - } - - uint32_t offsetByte = (uint64_t)ptr - (uint64_t)__hip_device_heap; - uint32_t offsetPage = offsetByte / __HIP_SIZE_OF_PAGE; - - while (__hip_device_page_flag[offsetPage] != 0) { - if (__hip_device_page_flag[offsetPage] == 2) { - __hip_device_page_flag[offsetPage] = 0; - offsetPage++; - break; - } else { - __hip_device_page_flag[offsetPage] = 0; - offsetPage++; - } - } - - return nullptr; -} - -#endif - -#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_MEMORY_H diff --git a/src/hipamd/include/hip/amd_detail/hip_runtime_prof.h b/src/hipamd/include/hip/amd_detail/hip_runtime_prof.h deleted file mode 100644 index a45962d459..0000000000 --- a/src/hipamd/include/hip/amd_detail/hip_runtime_prof.h +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright (c) 2019 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H - -// HIP ROCclr Op IDs enumeration -enum HipVdiOpId { - kHipVdiOpIdDispatch = 0, - kHipVdiOpIdCopy = 1, - kHipVdiOpIdBarrier = 2, - kHipVdiOpIdNumber = 3 -}; - -// Types of ROCclr commands -enum HipVdiCommandKind { - kHipVdiCommandKernel = 0x11F0, - kHipVdiMemcpyDeviceToHost = 0x11F3, - kHipHipVdiMemcpyHostToDevice = 0x11F4, - kHipVdiMemcpyDeviceToDevice = 0x11F5, - kHipVidMemcpyDeviceToHostRect = 0x1201, - kHipVdiMemcpyHostToDeviceRect = 0x1202, - kHipVdiMemcpyDeviceToDeviceRect = 0x1203, - kHipVdiFillMemory = 0x1207, -}; - -/** - * @brief Initializes activity callback - * - * @param [input] id_callback Event ID callback function - * @param [input] op_callback Event operation callback function - * @param [input] arg Arguments passed into callback - * - * @returns None - */ -void hipInitActivityCallback(void* id_callback, void* op_callback, void* arg); - -/** - * @brief Enables activity callback - * - * @param [input] op Operation, which will trigger a callback (@see HipVdiOpId) - * @param [input] enable Enable state for the callback - * - * @returns True if successful - */ -bool hipEnableActivityCallback(uint32_t op, bool enable); - -/** - * @brief Returns the description string for the operation kind - * - * @param [input] id Command kind id (@see HipVdiCommandKind) - * - * @returns A pointer to a const string with the command description - */ -const char* hipGetCmdName(uint32_t id); - -#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H - diff --git a/src/hipamd/include/hip/amd_detail/host_defines.h b/src/hipamd/include/hip/amd_detail/host_defines.h deleted file mode 100644 index 7f1075acc1..0000000000 --- a/src/hipamd/include/hip/amd_detail/host_defines.h +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/host_defines.h - * @brief TODO-doc - */ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HOST_DEFINES_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_HOST_DEFINES_H - -// The follow macro should be removed after upstream updation. -// It's defined here for workarround of rocThrust building failure. -#define HIP_INCLUDE_HIP_HCC_DETAIL_HOST_DEFINES_H - -// Add guard to Generic Grid Launch method -#ifndef GENERIC_GRID_LAUNCH -#define GENERIC_GRID_LAUNCH 1 -#endif - -#if defined(__clang__) && defined(__HIP__) - -#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -#define __host__ __attribute__((host)) -#define __device__ __attribute__((device)) -#define __global__ __attribute__((global)) -#define __shared__ __attribute__((shared)) -#define __constant__ __attribute__((constant)) -#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ - -#define __noinline__ __attribute__((noinline)) -#define __forceinline__ inline __attribute__((always_inline)) - -#else - -// Non-HCC compiler -/** - * Function and kernel markers - */ -#define __host__ -#define __device__ - -#define __global__ - -#define __noinline__ -#define __forceinline__ inline - -#define __shared__ -#define __constant__ - -#endif - -#endif diff --git a/src/hipamd/include/hip/amd_detail/hsa_helpers.hpp b/src/hipamd/include/hip/amd_detail/hsa_helpers.hpp deleted file mode 100644 index af4f0c93ab..0000000000 --- a/src/hipamd/include/hip/amd_detail/hsa_helpers.hpp +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ -#pragma once - -#include - -#include -#include -#include - -namespace hip_impl { -inline void* address(hsa_executable_symbol_t x) { - void* r = nullptr; - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, &r); - - return r; -} - -inline hsa_agent_t agent(hsa_executable_symbol_t x) { - hsa_agent_t r = {}; - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_AGENT, &r); - - return r; -} - -inline std::uint32_t group_size(hsa_executable_symbol_t x) { - std::uint32_t r = 0u; - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_GROUP_SEGMENT_SIZE, &r); - - return r; -} - -inline hsa_isa_t isa(hsa_agent_t x) { - hsa_isa_t r = {}; - hsa_agent_iterate_isas(x, - [](hsa_isa_t i, void* o) { - *static_cast(o) = i; // Pick the first. - - return HSA_STATUS_INFO_BREAK; - }, - &r); - - return r; -} - -inline std::uint64_t kernel_object(hsa_executable_symbol_t x) { - std::uint64_t r = 0u; - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &r); - - return r; -} - -inline std::string name(hsa_executable_symbol_t x) { - std::uint32_t sz = 0u; - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_NAME_LENGTH, &sz); - - std::string r(sz, '\0'); - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_NAME, &r.front()); - - return r; -} - -inline std::uint32_t private_size(hsa_executable_symbol_t x) { - std::uint32_t r = 0u; - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_PRIVATE_SEGMENT_SIZE, &r); - - return r; -} - -inline std::uint32_t size(hsa_executable_symbol_t x) { - std::uint32_t r = 0; - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_SIZE, &r); - - return r; -} - -inline hsa_symbol_kind_t type(hsa_executable_symbol_t x) { - hsa_symbol_kind_t r = {}; - hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_TYPE, &r); - - return r; -} -} // namespace hip_impl \ No newline at end of file diff --git a/src/hipamd/include/hip/amd_detail/llvm_intrinsics.h b/src/hipamd/include/hip/amd_detail/llvm_intrinsics.h deleted file mode 100644 index c100b846c0..0000000000 --- a/src/hipamd/include/hip/amd_detail/llvm_intrinsics.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -/** - * @file amd_detail/llvm_intrinsics.h - * @brief Contains declarations for wrapper functions for llvm intrinsics - * like llvm.amdgcn.s.barrier. - */ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_LLVM_INTRINSICS_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_LLVM_INTRINSICS_H - -#include "hip/amd_detail/host_defines.h" - -// FIXME: These should all be removed and proper builtins used. -__device__ -unsigned __llvm_amdgcn_groupstaticsize() __asm("llvm.amdgcn.groupstaticsize"); - -__device__ -int __llvm_amdgcn_ds_swizzle(int index, int pattern) __asm("llvm.amdgcn.ds.swizzle"); - -#endif diff --git a/src/hipamd/include/hip/amd_detail/macro_based_grid_launch.hpp b/src/hipamd/include/hip/amd_detail/macro_based_grid_launch.hpp deleted file mode 100644 index 96d449b213..0000000000 --- a/src/hipamd/include/hip/amd_detail/macro_based_grid_launch.hpp +++ /dev/null @@ -1,798 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#include "concepts.hpp" -#include "helpers.hpp" - -#include "hc.hpp" -#include "hip/hip_ext.h" -#include "hip_runtime.h" - -#include -#include -#include -#include -#include - -namespace hip_impl { -namespace { -struct New_grid_launch_tag {}; -struct Old_grid_launch_tag {}; - -template -class RAII_guard { - D dtor_; - - public: - RAII_guard() = default; - - RAII_guard(const C& ctor, D dtor) : dtor_{std::move(dtor)} { ctor(); } - - RAII_guard(const RAII_guard&) = default; - RAII_guard(RAII_guard&&) = default; - - RAII_guard& operator=(const RAII_guard&) = default; - RAII_guard& operator=(RAII_guard&&) = default; - - ~RAII_guard() { dtor_(); } -}; - -template -RAII_guard make_RAII_guard(const C& ctor, D dtor) { - return RAII_guard{ctor, std::move(dtor)}; -} - -template -using is_new_grid_launch_t = typename std::conditional{}, New_grid_launch_tag, - Old_grid_launch_tag>::type; -} // namespace - -// TODO: - dispatch rank should be derived from the domain dimensions passed -// in, and not always assumed to be 3; - -template -requires(Domain == - {Ts...}) inline void grid_launch_hip_impl_(New_grid_launch_tag, dim3 num_blocks, - dim3 dim_blocks, int group_mem_bytes, - const hc::accelerator_view& acc_v, K k) { - const auto d = - hc::extent<3>{num_blocks.z * dim_blocks.z, num_blocks.y * dim_blocks.y, - num_blocks.x * dim_blocks.x} - .tile_with_dynamic(dim_blocks.z, dim_blocks.y, dim_blocks.x, group_mem_bytes); - - try { - hc::parallel_for_each(acc_v, d, k); - } catch (std::exception& ex) { - std::cerr << "Failed in " << __func__ << ", with exception: " << ex.what() << std::endl; - hip_throw(ex); - } -} - -// TODO: these are workarounds, they should be removed. - -hc::accelerator_view lock_stream_hip_(hipStream_t&, void*&); -void print_prelaunch_trace_(const char*, dim3, dim3, int, hipStream_t); -void unlock_stream_hip_(hipStream_t, void*, const char*, hc::accelerator_view*); - -template -requires(Domain == {Ts...}) inline void grid_launch_hip_impl_(New_grid_launch_tag, - dim3 num_blocks, dim3 dim_blocks, - int group_mem_bytes, - hipStream_t stream, - const char* kernel_name, K k) { - void* lck_stream = nullptr; - auto acc_v = lock_stream_hip_(stream, lck_stream); - auto stream_guard = - make_RAII_guard(std::bind(print_prelaunch_trace_, kernel_name, num_blocks, dim_blocks, - group_mem_bytes, stream), - std::bind(unlock_stream_hip_, stream, lck_stream, kernel_name, &acc_v)); - - try { - grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks), - group_mem_bytes, acc_v, std::move(k)); - } catch (std::exception& ex) { - std::cerr << "Failed in " << __func__ << ", with exception: " << ex.what() << std::endl; - hip_throw(ex); - } -} - -template -requires(Domain == - {hipLaunchParm, Ts...}) inline void grid_launch_hip_impl_(Old_grid_launch_tag, - dim3 num_blocks, dim3 dim_blocks, - int group_mem_bytes, - hipStream_t stream, K k) { - grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks), - group_mem_bytes, std::move(stream), std::move(k)); -} - -template -requires(Domain == {hipLaunchParm, Ts...}) inline void grid_launch_hip_impl_( - Old_grid_launch_tag, dim3 num_blocks, dim3 dim_blocks, int group_mem_bytes, hipStream_t stream, - const char* kernel_name, K k) { - grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks), - group_mem_bytes, std::move(stream), kernel_name, std::move(k)); -} - -template -requires(Domain == {Ts...}) inline std::enable_if_t< - !std::is_function::value> grid_launch_hip_(dim3 num_blocks, dim3 dim_blocks, - int group_mem_bytes, hipStream_t stream, - const char* kernel_name, K k) { - grid_launch_hip_impl_(is_new_grid_launch_t{}, std::move(num_blocks), - std::move(dim_blocks), group_mem_bytes, std::move(stream), kernel_name, - std::move(k)); -} - -template -requires(Domain == {Ts...}) inline std::enable_if_t< - !std::is_function::value> grid_launch_hip_(dim3 num_blocks, dim3 dim_blocks, - int group_mem_bytes, hipStream_t stream, K k) { - grid_launch_hip_impl_(is_new_grid_launch_t{}, std::move(num_blocks), - std::move(dim_blocks), group_mem_bytes, std::move(stream), std::move(k)); -} - -// TODO: these are temporary and purposefully noisy and disruptive. -#define make_kernel_name_hip(k, n) \ - HIP_kernel_functor_name_begin##_##k##_##HIP_kernel_functor_name_end##_##n - -#define make_kernel_functor_hip_30(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \ - p22, p23, p24, p25, p26, p27) \ - struct make_kernel_name_hip(function_name, 28) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - std::decay_t _p20_; \ - std::decay_t _p21_; \ - std::decay_t _p22_; \ - std::decay_t _p23_; \ - std::decay_t _p24_; \ - std::decay_t _p25_; \ - std::decay_t _p26_; \ - std::decay_t _p27_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \ - _p22_, _p23_, _p24_, _p25_, _p26_, _p27_); \ - } \ - } -#define make_kernel_functor_hip_29(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \ - p22, p23, p24, p25, p26) \ - struct make_kernel_name_hip(function_name, 27) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - std::decay_t _p20_; \ - std::decay_t _p21_; \ - std::decay_t _p22_; \ - std::decay_t _p23_; \ - std::decay_t _p24_; \ - std::decay_t _p25_; \ - std::decay_t _p26_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \ - _p22_, _p23_, _p24_, _p25_, _p26_); \ - } \ - } -#define make_kernel_functor_hip_28(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \ - p22, p23, p24, p25) \ - struct make_kernel_name_hip(function_name, 26) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - std::decay_t _p20_; \ - std::decay_t _p21_; \ - std::decay_t _p22_; \ - std::decay_t _p23_; \ - std::decay_t _p24_; \ - std::decay_t _p25_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \ - _p22_, _p23_, _p24_, _p25_); \ - } \ - } -#define make_kernel_functor_hip_27(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \ - p22, p23, p24) \ - struct make_kernel_name_hip(function_name, 25) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - std::decay_t _p20_; \ - std::decay_t _p21_; \ - std::decay_t _p22_; \ - std::decay_t _p23_; \ - std::decay_t _p24_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \ - _p22_, _p23_, _p24_); \ - } \ - } -#define make_kernel_functor_hip_26(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \ - p22, p23) \ - struct make_kernel_name_hip(function_name, 24) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - std::decay_t _p20_; \ - std::decay_t _p21_; \ - std::decay_t _p22_; \ - std::decay_t _p23_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \ - _p22_, _p23_); \ - } \ - } -#define make_kernel_functor_hip_25(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \ - p22) \ - struct make_kernel_name_hip(function_name, 23) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - std::decay_t _p20_; \ - std::decay_t _p21_; \ - std::decay_t _p22_; \ - __attribute__((used, flatten)) void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \ - _p22_); \ - } \ - } -#define make_kernel_functor_hip_24(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21) \ - struct make_kernel_name_hip(function_name, 22) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - std::decay_t _p20_; \ - std::decay_t _p21_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_); \ - } \ - } -#define make_kernel_functor_hip_23(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20) \ - struct make_kernel_name_hip(function_name, 21) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - std::decay_t _p20_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_); \ - } \ - } -#define make_kernel_functor_hip_22(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19) \ - struct make_kernel_name_hip(function_name, 20) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - std::decay_t _p19_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_); \ - } \ - } -#define make_kernel_functor_hip_21(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17, p18) \ - struct make_kernel_name_hip(function_name, 19) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - std::decay_t _p18_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_); \ - } \ - } -#define make_kernel_functor_hip_20(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16, p17) \ - struct make_kernel_name_hip(function_name, 18) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - std::decay_t _p17_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_, _p17_); \ - } \ - } -#define make_kernel_functor_hip_19(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16) \ - struct make_kernel_name_hip(function_name, 17) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - std::decay_t _p16_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_, _p16_); \ - } \ - } -#define make_kernel_functor_hip_18(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15) \ - struct make_kernel_name_hip(function_name, 16) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - std::decay_t _p15_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_, _p15_); \ - } \ - } -#define make_kernel_functor_hip_17(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14) \ - struct make_kernel_name_hip(function_name, 15) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - std::decay_t _p14_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_, _p14_); \ - } \ - } -#define make_kernel_functor_hip_16(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13) \ - struct make_kernel_name_hip(function_name, 14) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - std::decay_t _p13_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_, _p13_); \ - } \ - } -#define make_kernel_functor_hip_15(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12) \ - struct make_kernel_name_hip(function_name, 13) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - std::decay_t _p12_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \ - _p12_); \ - } \ - } -#define make_kernel_functor_hip_14(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11) \ - struct make_kernel_name_hip(function_name, 12) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - std::decay_t _p11_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_); \ - } \ - } -#define make_kernel_functor_hip_13(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10) \ - struct make_kernel_name_hip(function_name, 11) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - std::decay_t _p10_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { \ - kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_); \ - } \ - } -#define make_kernel_functor_hip_12(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \ - p9) \ - struct make_kernel_name_hip(function_name, 10) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - std::decay_t _p9_; \ - void operator()(const hc::tiled_index<3>&) const \ - [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_); } \ - } -#define make_kernel_functor_hip_11(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8) \ - struct make_kernel_name_hip(function_name, 9) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - std::decay_t _p8_; \ - void operator()(const hc::tiled_index<3>&) const \ - [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_); } \ - } -#define make_kernel_functor_hip_10(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7) \ - struct make_kernel_name_hip(function_name, 8) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - std::decay_t _p7_; \ - void operator()(const hc::tiled_index<3>&) const \ - [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_); } \ - } -#define make_kernel_functor_hip_9(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6) \ - struct make_kernel_name_hip(function_name, 7) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - std::decay_t _p6_; \ - void operator()(const hc::tiled_index<3>&) const \ - [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_); } \ - } -#define make_kernel_functor_hip_8(function_name, kernel_name, p0, p1, p2, p3, p4, p5) \ - struct make_kernel_name_hip(function_name, 6) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - std::decay_t _p5_; \ - void operator()(const hc::tiled_index<3>&) const \ - [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_); } \ - } -#define make_kernel_functor_hip_7(function_name, kernel_name, p0, p1, p2, p3, p4) \ - struct make_kernel_name_hip(function_name, 5) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - std::decay_t _p4_; \ - void operator()(const hc::tiled_index<3>&) const \ - [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_); } \ - } -#define make_kernel_functor_hip_6(function_name, kernel_name, p0, p1, p2, p3) \ - struct make_kernel_name_hip(function_name, 4) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - std::decay_t _p3_; \ - void operator()(const hc::tiled_index<3>&) const \ - [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_); } \ - } -#define make_kernel_functor_hip_5(function_name, kernel_name, p0, p1, p2) \ - struct make_kernel_name_hip(function_name, 3) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - std::decay_t _p2_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_, _p1_, _p2_); } \ - } -#define make_kernel_functor_hip_4(function_name, kernel_name, p0, p1) \ - struct make_kernel_name_hip(function_name, 2) { \ - std::decay_t _p0_; \ - std::decay_t _p1_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_, _p1_); } \ - } -#define fofo(f, n) kernel_prefix_hip##f##kernel_suffix_hip##n -#define make_kernel_functor_hip_3(function_name, kernel_name, p0) \ - struct make_kernel_name_hip(function_name, 1) { \ - std::decay_t _p0_; \ - void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_); } \ - } -#define make_kernel_functor_hip_2(function_name, kernel_name) \ - struct make_kernel_name_hip(function_name, 0) { \ - void operator()(const hc::tiled_index<3>&)[[hc]] { return kernel_name(hipLaunchParm{}); } \ - } -#define make_kernel_functor_hip_1(...) -#define make_kernel_functor_hip_0(...) -#define make_kernel_functor_hip_(...) overload_macro_hip_(make_kernel_functor_hip_, __VA_ARGS__) - - -#define hipLaunchNamedKernelGGL(function_name, kernel_name, num_blocks, dim_blocks, \ - group_mem_bytes, stream, ...) \ - do { \ - make_kernel_functor_hip_(function_name, kernel_name, __VA_ARGS__) \ - hip_kernel_functor_impl_{__VA_ARGS__}; \ - hip_impl::grid_launch_hip_(num_blocks, dim_blocks, group_mem_bytes, stream, #kernel_name, \ - hip_kernel_functor_impl_); \ - } while (0) - -#define hipLaunchKernelGGL(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, ...) \ - do { \ - hipLaunchNamedKernelGGL(unnamed, kernel_name, num_blocks, dim_blocks, group_mem_bytes, \ - stream, ##__VA_ARGS__); \ - } while (0) - -#define hipLaunchKernel(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, ...) \ - do { \ - hipLaunchKernelGGL(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, \ - hipLaunchParm{}, ##__VA_ARGS__); \ - } while (0) -} // namespace hip_impl diff --git a/src/hipamd/include/hip/amd_detail/math_fwd.h b/src/hipamd/include/hip/amd_detail/math_fwd.h deleted file mode 100644 index ac46d537a8..0000000000 --- a/src/hipamd/include/hip/amd_detail/math_fwd.h +++ /dev/null @@ -1,714 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#include "host_defines.h" -#if defined(__cplusplus) - extern "C" { -#endif - -// DOT FUNCTIONS -#if __HIP_CLANG_ONLY__ -__device__ -__attribute__((const)) -int __ockl_sdot2( - HIP_vector_base::Native_vec_, - HIP_vector_base::Native_vec_, - int, bool); - -__device__ -__attribute__((const)) -unsigned int __ockl_udot2( - HIP_vector_base::Native_vec_, - HIP_vector_base::Native_vec_, - unsigned int, bool); - -__device__ -__attribute__((const)) -int __ockl_sdot4( - HIP_vector_base::Native_vec_, - HIP_vector_base::Native_vec_, - int, bool); - -__device__ -__attribute__((const)) -unsigned int __ockl_udot4( - HIP_vector_base::Native_vec_, - HIP_vector_base::Native_vec_, - unsigned int, bool); - -__device__ -__attribute__((const)) -int __ockl_sdot8(int, int, int, bool); - -__device__ -__attribute__((const)) -unsigned int __ockl_udot8(unsigned int, unsigned int, unsigned int, bool); -#endif - -#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ -// BEGIN FLOAT -__device__ -__attribute__((const)) -float __ocml_acos_f32(float); -__device__ -__attribute__((pure)) -float __ocml_acosh_f32(float); -__device__ -__attribute__((const)) -float __ocml_asin_f32(float); -__device__ -__attribute__((pure)) -float __ocml_asinh_f32(float); -__device__ -__attribute__((const)) -float __ocml_atan2_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_atan_f32(float); -__device__ -__attribute__((pure)) -float __ocml_atanh_f32(float); -__device__ -__attribute__((pure)) -float __ocml_cbrt_f32(float); -__device__ -__attribute__((const)) -float __ocml_ceil_f32(float); -__device__ -__attribute__((const)) -__device__ -float __ocml_copysign_f32(float, float); -__device__ -float __ocml_cos_f32(float); -__device__ -float __ocml_native_cos_f32(float); -__device__ -__attribute__((pure)) -__device__ -float __ocml_cosh_f32(float); -__device__ -float __ocml_cospi_f32(float); -__device__ -float __ocml_i0_f32(float); -__device__ -float __ocml_i1_f32(float); -__device__ -__attribute__((pure)) -float __ocml_erfc_f32(float); -__device__ -__attribute__((pure)) -float __ocml_erfcinv_f32(float); -__device__ -__attribute__((pure)) -float __ocml_erfcx_f32(float); -__device__ -__attribute__((pure)) -float __ocml_erf_f32(float); -__device__ -__attribute__((pure)) -float __ocml_erfinv_f32(float); -__device__ -__attribute__((pure)) -float __ocml_exp10_f32(float); -__device__ -__attribute__((pure)) -float __ocml_native_exp10_f32(float); -__device__ -__attribute__((pure)) -float __ocml_exp2_f32(float); -__device__ -__attribute__((pure)) -float __ocml_exp_f32(float); -__device__ -__attribute__((pure)) -float __ocml_native_exp_f32(float); -__device__ -__attribute__((pure)) -float __ocml_expm1_f32(float); -__device__ -__attribute__((const)) -float __ocml_fabs_f32(float); -__device__ -__attribute__((const)) -float __ocml_fdim_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_floor_f32(float); -__device__ -__attribute__((const)) -float __ocml_fma_f32(float, float, float); -__device__ -__attribute__((const)) -float __ocml_fmax_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_fmin_f32(float, float); -__device__ -__attribute__((const)) -__device__ -float __ocml_fmod_f32(float, float); -__device__ -float __ocml_frexp_f32(float, __attribute__((address_space(5))) int*); -__device__ -__attribute__((const)) -float __ocml_hypot_f32(float, float); -__device__ -__attribute__((const)) -int __ocml_ilogb_f32(float); -__device__ -__attribute__((const)) -int __ocml_isfinite_f32(float); -__device__ -__attribute__((const)) -int __ocml_isinf_f32(float); -__device__ -__attribute__((const)) -int __ocml_isnan_f32(float); -__device__ -float __ocml_j0_f32(float); -__device__ -float __ocml_j1_f32(float); -__device__ -__attribute__((const)) -float __ocml_ldexp_f32(float, int); -__device__ -float __ocml_lgamma_f32(float); -__device__ -__attribute__((pure)) -float __ocml_log10_f32(float); -__device__ -__attribute__((pure)) -float __ocml_native_log10_f32(float); -__device__ -__attribute__((pure)) -float __ocml_log1p_f32(float); -__device__ -__attribute__((pure)) -float __ocml_log2_f32(float); -__device__ -__attribute__((pure)) -float __ocml_native_log2_f32(float); -__device__ -__attribute__((const)) -float __ocml_logb_f32(float); -__device__ -__attribute__((pure)) -float __ocml_log_f32(float); -__device__ -__attribute__((pure)) -float __ocml_native_log_f32(float); -__device__ -float __ocml_modf_f32(float, __attribute__((address_space(5))) float*); -__device__ -__attribute__((const)) -float __ocml_nearbyint_f32(float); -__device__ -__attribute__((const)) -float __ocml_nextafter_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_len3_f32(float, float, float); -__device__ -__attribute__((const)) -float __ocml_len4_f32(float, float, float, float); -__device__ -__attribute__((pure)) -float __ocml_ncdf_f32(float); -__device__ -__attribute__((pure)) -float __ocml_ncdfinv_f32(float); -__device__ -__attribute__((pure)) -float __ocml_pow_f32(float, float); -__device__ -__attribute__((pure)) -float __ocml_pown_f32(float, int); -__device__ -__attribute__((pure)) -float __ocml_rcbrt_f32(float); -__device__ -__attribute__((const)) -float __ocml_remainder_f32(float, float); -__device__ -float __ocml_remquo_f32(float, float, __attribute__((address_space(5))) int*); -__device__ -__attribute__((const)) -float __ocml_rhypot_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_rint_f32(float); -__device__ -__attribute__((const)) -float __ocml_rlen3_f32(float, float, float); -__device__ -__attribute__((const)) -float __ocml_rlen4_f32(float, float, float, float); -__device__ -__attribute__((const)) -float __ocml_round_f32(float); -__device__ -__attribute__((pure)) -float __ocml_rsqrt_f32(float); -__device__ -__attribute__((const)) -float __ocml_scalb_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_scalbn_f32(float, int); -__device__ -__attribute__((const)) -int __ocml_signbit_f32(float); -__device__ -float __ocml_sincos_f32(float, __attribute__((address_space(5))) float*); -__device__ -float __ocml_sincospi_f32(float, __attribute__((address_space(5))) float*); -__device__ -float __ocml_sin_f32(float); -__device__ -float __ocml_native_sin_f32(float); -__device__ -__attribute__((pure)) -float __ocml_sinh_f32(float); -__device__ -float __ocml_sinpi_f32(float); -__device__ -__attribute__((const)) -float __ocml_sqrt_f32(float); -__device__ -__attribute__((const)) -float __ocml_native_sqrt_f32(float); -__device__ -float __ocml_tan_f32(float); -__device__ -__attribute__((pure)) -float __ocml_tanh_f32(float); -__device__ -float __ocml_tgamma_f32(float); -__device__ -__attribute__((const)) -float __ocml_trunc_f32(float); -__device__ -float __ocml_y0_f32(float); -__device__ -float __ocml_y1_f32(float); - -// BEGIN INTRINSICS -__device__ -__attribute__((const)) -float __ocml_add_rte_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_add_rtn_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_add_rtp_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_add_rtz_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_sub_rte_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_sub_rtn_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_sub_rtp_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_sub_rtz_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_mul_rte_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_mul_rtn_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_mul_rtp_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_mul_rtz_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_div_rte_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_div_rtn_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_div_rtp_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_div_rtz_f32(float, float); -__device__ -__attribute__((const)) -float __ocml_sqrt_rte_f32(float); -__device__ -__attribute__((const)) -float __ocml_sqrt_rtn_f32(float); -__device__ -__attribute__((const)) -float __ocml_sqrt_rtp_f32(float); -__device__ -__attribute__((const)) -float __ocml_sqrt_rtz_f32(float); -__device__ -__attribute__((const)) -float __ocml_fma_rte_f32(float, float, float); -__device__ -__attribute__((const)) -float __ocml_fma_rtn_f32(float, float, float); -__device__ -__attribute__((const)) -float __ocml_fma_rtp_f32(float, float, float); -__device__ -__attribute__((const)) -float __ocml_fma_rtz_f32(float, float, float); - -__device__ -__attribute__((const)) -float __llvm_amdgcn_cos_f32(float) __asm("llvm.amdgcn.cos.f32"); -__device__ -__attribute__((const)) -float __llvm_amdgcn_rcp_f32(float) __asm("llvm.amdgcn.rcp.f32"); -__device__ -__attribute__((const)) -float __llvm_amdgcn_rsq_f32(float) __asm("llvm.amdgcn.rsq.f32"); -__device__ -__attribute__((const)) -float __llvm_amdgcn_sin_f32(float) __asm("llvm.amdgcn.sin.f32"); -// END INTRINSICS -// END FLOAT - -// BEGIN DOUBLE -__device__ -__attribute__((const)) -double __ocml_acos_f64(double); -__device__ -__attribute__((pure)) -double __ocml_acosh_f64(double); -__device__ -__attribute__((const)) -double __ocml_asin_f64(double); -__device__ -__attribute__((pure)) -double __ocml_asinh_f64(double); -__device__ -__attribute__((const)) -double __ocml_atan2_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_atan_f64(double); -__device__ -__attribute__((pure)) -double __ocml_atanh_f64(double); -__device__ -__attribute__((pure)) -double __ocml_cbrt_f64(double); -__device__ -__attribute__((const)) -double __ocml_ceil_f64(double); -__device__ -__attribute__((const)) -double __ocml_copysign_f64(double, double); -__device__ -double __ocml_cos_f64(double); -__device__ -__attribute__((pure)) -double __ocml_cosh_f64(double); -__device__ -double __ocml_cospi_f64(double); -__device__ -double __ocml_i0_f64(double); -__device__ -double __ocml_i1_f64(double); -__device__ -__attribute__((pure)) -double __ocml_erfc_f64(double); -__device__ -__attribute__((pure)) -double __ocml_erfcinv_f64(double); -__device__ -__attribute__((pure)) -double __ocml_erfcx_f64(double); -__device__ -__attribute__((pure)) -double __ocml_erf_f64(double); -__device__ -__attribute__((pure)) -double __ocml_erfinv_f64(double); -__device__ -__attribute__((pure)) -double __ocml_exp10_f64(double); -__device__ -__attribute__((pure)) -double __ocml_exp2_f64(double); -__device__ -__attribute__((pure)) -double __ocml_exp_f64(double); -__device__ -__attribute__((pure)) -double __ocml_expm1_f64(double); -__device__ -__attribute__((const)) -double __ocml_fabs_f64(double); -__device__ -__attribute__((const)) -double __ocml_fdim_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_floor_f64(double); -__device__ -__attribute__((const)) -double __ocml_fma_f64(double, double, double); -__device__ -__attribute__((const)) -double __ocml_fmax_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_fmin_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_fmod_f64(double, double); -__device__ -double __ocml_frexp_f64(double, __attribute__((address_space(5))) int*); -__device__ -__attribute__((const)) -double __ocml_hypot_f64(double, double); -__device__ -__attribute__((const)) -int __ocml_ilogb_f64(double); -__device__ -__attribute__((const)) -int __ocml_isfinite_f64(double); -__device__ -__attribute__((const)) -int __ocml_isinf_f64(double); -__device__ -__attribute__((const)) -int __ocml_isnan_f64(double); -__device__ -double __ocml_j0_f64(double); -__device__ -double __ocml_j1_f64(double); -__device__ -__attribute__((const)) -double __ocml_ldexp_f64(double, int); -__device__ -double __ocml_lgamma_f64(double); -__device__ -__attribute__((pure)) -double __ocml_log10_f64(double); -__device__ -__attribute__((pure)) -double __ocml_log1p_f64(double); -__device__ -__attribute__((pure)) -double __ocml_log2_f64(double); -__device__ -__attribute__((const)) -double __ocml_logb_f64(double); -__device__ -__attribute__((pure)) -double __ocml_log_f64(double); -__device__ -double __ocml_modf_f64(double, __attribute__((address_space(5))) double*); -__device__ -__attribute__((const)) -double __ocml_nearbyint_f64(double); -__device__ -__attribute__((const)) -double __ocml_nextafter_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_len3_f64(double, double, double); -__device__ -__attribute__((const)) -double __ocml_len4_f64(double, double, double, double); -__device__ -__attribute__((pure)) -double __ocml_ncdf_f64(double); -__device__ -__attribute__((pure)) -double __ocml_ncdfinv_f64(double); -__device__ -__attribute__((pure)) -double __ocml_pow_f64(double, double); -__device__ -__attribute__((pure)) -double __ocml_pown_f64(double, int); -__device__ -__attribute__((pure)) -double __ocml_rcbrt_f64(double); -__device__ -__attribute__((const)) -double __ocml_remainder_f64(double, double); -__device__ -double __ocml_remquo_f64( - double, double, __attribute__((address_space(5))) int*); -__device__ -__attribute__((const)) -double __ocml_rhypot_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_rint_f64(double); -__device__ -__attribute__((const)) -double __ocml_rlen3_f64(double, double, double); -__device__ -__attribute__((const)) -double __ocml_rlen4_f64(double, double, double, double); -__device__ -__attribute__((const)) -double __ocml_round_f64(double); -__device__ -__attribute__((pure)) -double __ocml_rsqrt_f64(double); -__device__ -__attribute__((const)) -double __ocml_scalb_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_scalbn_f64(double, int); -__device__ -__attribute__((const)) -int __ocml_signbit_f64(double); -__device__ -double __ocml_sincos_f64(double, __attribute__((address_space(5))) double*); -__device__ -double __ocml_sincospi_f64(double, __attribute__((address_space(5))) double*); -__device__ -double __ocml_sin_f64(double); -__device__ -__attribute__((pure)) -double __ocml_sinh_f64(double); -__device__ -double __ocml_sinpi_f64(double); -__device__ -__attribute__((const)) -double __ocml_sqrt_f64(double); -__device__ -double __ocml_tan_f64(double); -__device__ -__attribute__((pure)) -double __ocml_tanh_f64(double); -__device__ -double __ocml_tgamma_f64(double); -__device__ -__attribute__((const)) -double __ocml_trunc_f64(double); -__device__ -double __ocml_y0_f64(double); -__device__ -double __ocml_y1_f64(double); - -// BEGIN INTRINSICS -__device__ -__attribute__((const)) -double __ocml_add_rte_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_add_rtn_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_add_rtp_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_add_rtz_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_sub_rte_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_sub_rtn_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_sub_rtp_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_sub_rtz_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_mul_rte_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_mul_rtn_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_mul_rtp_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_mul_rtz_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_div_rte_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_div_rtn_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_div_rtp_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_div_rtz_f64(double, double); -__device__ -__attribute__((const)) -double __ocml_sqrt_rte_f64(double); -__device__ -__attribute__((const)) -double __ocml_sqrt_rtn_f64(double); -__device__ -__attribute__((const)) -double __ocml_sqrt_rtp_f64(double); -__device__ -__attribute__((const)) -double __ocml_sqrt_rtz_f64(double); -__device__ -__attribute__((const)) -double __ocml_fma_rte_f64(double, double, double); -__device__ -__attribute__((const)) -double __ocml_fma_rtn_f64(double, double, double); -__device__ -__attribute__((const)) -double __ocml_fma_rtp_f64(double, double, double); -__device__ -__attribute__((const)) -double __ocml_fma_rtz_f64(double, double, double); - -__device__ -__attribute__((const)) -double __llvm_amdgcn_rcp_f64(double) __asm("llvm.amdgcn.rcp.f64"); -__device__ -__attribute__((const)) -double __llvm_amdgcn_rsq_f64(double) __asm("llvm.amdgcn.rsq.f64"); -// END INTRINSICS -// END DOUBLE - -#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__ - -#if defined(__cplusplus) - } // extern "C" -#endif diff --git a/src/hipamd/include/hip/amd_detail/ockl_image.h b/src/hipamd/include/hip/amd_detail/ockl_image.h deleted file mode 100644 index b32b23fda0..0000000000 --- a/src/hipamd/include/hip/amd_detail/ockl_image.h +++ /dev/null @@ -1,135 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#include - -extern "C" { - -#define ADDRESS_SPACE_CONSTANT __attribute__((address_space(4))) - -__device__ float4::Native_vec_ __ockl_image_load_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c); - -__device__ float4::Native_vec_ __ockl_image_load_1Db(unsigned int ADDRESS_SPACE_CONSTANT*i, int c); - -__device__ float4::Native_vec_ __ockl_image_load_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_load_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_load_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_load_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_load_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f); - -__device__ float4::Native_vec_ __ockl_image_load_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f); - -__device__ float4::Native_vec_ __ockl_image_load_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l); - -__device__ float4::Native_vec_ __ockl_image_load_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l); - -__device__ float4::Native_vec_ __ockl_image_load_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l); - -__device__ float4::Native_vec_ __ockl_image_load_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l); - -__device__ float4::Native_vec_ __ockl_image_load_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l); - -__device__ float4::Native_vec_ __ockl_image_load_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, int l); - -__device__ float4::Native_vec_ __ockl_image_load_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, int l); - -__device__ void __ockl_image_store_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, float4::Native_vec_ p); - -__device__ void __ockl_image_store_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p); - -__device__ void __ockl_image_store_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p); - -__device__ void __ockl_image_store_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p); - -__device__ void __ockl_image_store_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p); - -__device__ void __ockl_image_store_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p); - -__device__ void __ockl_image_store_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p); - -__device__ void __ockl_image_store_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l, float4::Native_vec_ p); - -__device__ void __ockl_image_store_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p); - -__device__ void __ockl_image_store_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p); - -__device__ void __ockl_image_store_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p); - -__device__ void __ockl_image_store_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p); - -__device__ void __ockl_image_store_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p); - -__device__ void __ockl_image_store_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p); - -__device__ float4::Native_vec_ __ockl_image_sample_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c); - -__device__ float4::Native_vec_ __ockl_image_sample_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_sample_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_sample_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_sample_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_sample_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_sample_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_sample_grad_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float dx, float dy); - -__device__ float4::Native_vec_ __ockl_image_sample_grad_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float dx, float dy); - -__device__ float4::Native_vec_ __ockl_image_sample_grad_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy); - -__device__ float4::Native_vec_ __ockl_image_sample_grad_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy); - -__device__ float4::Native_vec_ __ockl_image_sample_grad_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float4::Native_vec_ dx, float4::Native_vec_ dy); - -__device__ float4::Native_vec_ __ockl_image_sample_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float l); - -__device__ float4::Native_vec_ __ockl_image_sample_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l); - -__device__ float4::Native_vec_ __ockl_image_sample_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l); - -__device__ float4::Native_vec_ __ockl_image_sample_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l); - -__device__ float4::Native_vec_ __ockl_image_sample_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l); - -__device__ float4::Native_vec_ __ockl_image_sample_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l); - -__device__ float4::Native_vec_ __ockl_image_sample_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l); - -__device__ float4::Native_vec_ __ockl_image_gather4r_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_gather4g_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_gather4b_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); - -__device__ float4::Native_vec_ __ockl_image_gather4a_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); - -}; \ No newline at end of file diff --git a/src/hipamd/include/hip/amd_detail/program_state.hpp b/src/hipamd/include/hip/amd_detail/program_state.hpp deleted file mode 100644 index 6128a4c158..0000000000 --- a/src/hipamd/include/hip/amd_detail/program_state.hpp +++ /dev/null @@ -1,107 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - -#include - -struct ihipModuleSymbol_t; -using hipFunction_t = ihipModuleSymbol_t*; - -namespace hip_impl { - -// This section contains internal APIs that -// needs to be exported -#ifdef __GNUC__ -#pragma GCC visibility push (default) -#endif - -struct kernarg_impl; -class kernarg { -public: - kernarg(); - kernarg(kernarg&&); - ~kernarg(); - std::uint8_t* data(); - std::size_t size(); - void reserve(std::size_t); - void resize(std::size_t); -private: - kernarg_impl* impl; -}; - -class kernargs_size_align; -class program_state_impl; -class program_state { -public: - program_state(); - ~program_state(); - program_state(const program_state&) = delete; - - hipFunction_t kernel_descriptor(std::uintptr_t, - hsa_agent_t); - - kernargs_size_align get_kernargs_size_align(std::uintptr_t); - hsa_executable_t load_executable(const char*, const size_t, - hsa_executable_t, - hsa_agent_t); - hsa_executable_t load_executable_no_copy(const char*, const size_t, - hsa_executable_t, - hsa_agent_t); - - void* global_addr_by_name(const char* name); - -private: - friend class agent_globals_impl; - program_state_impl* impl; -}; - -class kernargs_size_align { -public: - std::size_t size(std::size_t n) const; - std::size_t alignment(std::size_t n) const; - const void* getHandle() const {return handle;}; -private: - const void* handle; - friend kernargs_size_align program_state::get_kernargs_size_align(std::uintptr_t); -}; - -#ifdef __GNUC__ -#pragma GCC visibility pop -#endif - -inline -__attribute__((visibility("hidden"))) -program_state& get_program_state() { - static program_state ps; - return ps; -} -} // Namespace hip_impl. diff --git a/src/hipamd/include/hip/amd_detail/texture_fetch_functions.h b/src/hipamd/include/hip/amd_detail/texture_fetch_functions.h deleted file mode 100644 index 399e4fecf7..0000000000 --- a/src/hipamd/include/hip/amd_detail/texture_fetch_functions.h +++ /dev/null @@ -1,388 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#if defined(__cplusplus) - -#include -#include -#include - -#if !defined(__HIPCC_RTC__) -#include -#endif // !defined(__HIPCC_RTC__) - -#define TEXTURE_PARAMETERS_INIT \ - unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)t.textureObject; \ - unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; - -template -struct __hip_is_tex_channel_type -{ - static constexpr bool value = - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value; -}; - -template< - typename T, - unsigned int rank> -struct __hip_is_tex_channel_type> -{ - static constexpr bool value = - __hip_is_tex_channel_type::value && - ((rank == 1) || - (rank == 2) || - (rank == 4)); -}; - -template -struct __hip_is_tex_normalized_channel_type -{ - static constexpr bool value = - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value; -}; - -template< - typename T, - unsigned int rank> -struct __hip_is_tex_normalized_channel_type> -{ - static constexpr bool value = - __hip_is_tex_normalized_channel_type::value && - ((rank == 1) || - (rank == 2) || - (rank == 4)); -}; - -template < - typename T, - hipTextureReadMode readMode, - typename Enable = void> -struct __hip_tex_ret -{ - static_assert(std::is_same::value, "Invalid channel type!"); -}; - -template < - typename T, - hipTextureReadMode readMode> -using __hip_tex_ret_t = typename __hip_tex_ret::type; - -template -struct __hip_tex_ret< - T, - hipReadModeElementType, - typename std::enable_if<__hip_is_tex_channel_type::value, bool>::type> -{ - using type = T; -}; - -template< - typename T, - unsigned int rank> -struct __hip_tex_ret< - HIP_vector_type, - hipReadModeElementType, - typename std::enable_if<__hip_is_tex_channel_type>::value, bool>::type> -{ - using type = HIP_vector_type<__hip_tex_ret_t, rank>; -}; - -template -struct __hip_tex_ret< - T, - hipReadModeNormalizedFloat, - typename std::enable_if<__hip_is_tex_normalized_channel_type::value, bool>::type> -{ - using type = float; -}; - -template< - typename T, - unsigned int rank> -struct __hip_tex_ret< - HIP_vector_type, - hipReadModeNormalizedFloat, - typename std::enable_if<__hip_is_tex_normalized_channel_type>::value, bool>::type> -{ - using type = HIP_vector_type<__hip_tex_ret_t, rank>; -}; - -template -static __forceinline__ __device__ __hip_tex_ret_t tex1Dfetch(texture t, int x) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_load_1Db(i, x); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex1D(texture t, float x) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_1D(i, s, x); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex2D(texture t, float x, float y) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_2D(i, s, float2(x, y).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex1DLayered(texture t, float x, int layer) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex2DLayered(texture t, float x, float y, int layer) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex3D(texture t, float x, float y, float z) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t texCubemap(texture t, float x, float y, float z) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_CM(i, s, float4(x, y, z, 0.0f).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex1DLod(texture t, float x, float level) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_lod_1D(i, s, x, level); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex2DLod(texture t, float x, float y, float level) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex1DLayeredLod(texture t, float x, int layer, float level) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex2DLayeredLod(texture t, float x, float y, int layer, float level) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_lod_2Da(i, s, float4(x, y, layer, 0.0f).data, level); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex3DLod(texture t, float x, float y, float z, float level) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, level); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t texCubemapLod(texture t, float x, float y, float z, float level) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_lod_CM(i, s, float4(x, y, z, 0.0f).data, level); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayered(texture t, float x, float y, float z, int layer) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_CMa(i, s, float4(x, y, z, layer).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayeredLod(texture t, float x, float y, float z, int layer, float level) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_lod_CMa(i, s, float4(x, y, z, layer).data, level); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t texCubemapGrad(texture t, float x, float y, float z, float4 dPdx, float4 dPdy) -{ - TEXTURE_PARAMETERS_INIT; - // TODO missing in device libs. - // auto tmp = __ockl_image_sample_grad_CM(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); - // return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); - return {}; -} - -template -static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayeredGrad(texture t, float x, float y, float z, int layer, float4 dPdx, float4 dPdy) -{ - TEXTURE_PARAMETERS_INIT; - // TODO missing in device libs. - // auto tmp = __ockl_image_sample_grad_CMa(i, s, float4(x, y, z, layer).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); - // return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); - return {}; -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex1DGrad(texture t, float x, float dPdx, float dPdy) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex2DGrad(texture t, float x, float y, float2 dPdx, float2 dPdy) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex1DLayeredGrad(texture t, float x, int layer, float dPdx, float dPdy) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dPdx, dPdy); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex2DLayeredGrad(texture t, float x, float y, int layer, float2 dPdx, float2 dPdy) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template -static __forceinline__ __device__ __hip_tex_ret_t tex3DGrad(texture t, float x, float y, float z, float4 dPdx, float4 dPdy) -{ - TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); - return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); -} - -template < - typename T, - hipTextureReadMode readMode, - typename Enable = void> -struct __hip_tex2dgather_ret -{ - static_assert(std::is_same::value, "Invalid channel type!"); -}; - -template < - typename T, - hipTextureReadMode readMode> -using __hip_tex2dgather_ret_t = typename __hip_tex2dgather_ret::type; - -template -struct __hip_tex2dgather_ret< - T, - hipReadModeElementType, - typename std::enable_if<__hip_is_tex_channel_type::value, bool>::type> -{ - using type = HIP_vector_type; -}; - -template< - typename T, - unsigned int rank> -struct __hip_tex2dgather_ret< - HIP_vector_type, - hipReadModeElementType, - typename std::enable_if<__hip_is_tex_channel_type>::value, bool>::type> -{ - using type = HIP_vector_type; -}; - -template -struct __hip_tex2dgather_ret< - T, - hipReadModeNormalizedFloat, - typename std::enable_if<__hip_is_tex_normalized_channel_type::value, bool>::type> -{ - using type = float4; -}; - -template -static __forceinline__ __device__ __hip_tex2dgather_ret_t tex2Dgather(texture t, float x, float y, int comp=0) -{ - TEXTURE_PARAMETERS_INIT; - switch (comp) { - case 1: { - auto tmp = __ockl_image_gather4g_2D(i, s, float2(x, y).data); - return *reinterpret_cast<__hip_tex2dgather_ret_t*>(&tmp); - } - case 2: { - auto tmp = __ockl_image_gather4b_2D(i, s, float2(x, y).data); - return *reinterpret_cast<__hip_tex2dgather_ret_t*>(&tmp); - } - case 3: { - auto tmp = __ockl_image_gather4a_2D(i, s, float2(x, y).data); - return *reinterpret_cast<__hip_tex2dgather_ret_t*>(&tmp); - } - default: { - auto tmp = __ockl_image_gather4r_2D(i, s, float2(x, y).data); - return *reinterpret_cast<__hip_tex2dgather_ret_t*>(&tmp); - } - } - return {}; -} - -#endif diff --git a/src/hipamd/include/hip/amd_detail/texture_functions.h b/src/hipamd/include/hip/amd_detail/texture_functions.h deleted file mode 100644 index 5c2da3ef17..0000000000 --- a/src/hipamd/include/hip/amd_detail/texture_functions.h +++ /dev/null @@ -1,11102 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_TEXTURE_FUNCTIONS_H -#define HIP_INCLUDE_HIP_AMD_DETAIL_TEXTURE_FUNCTIONS_H -#include -#include - -#pragma push_macro("TYPEDEF_VECTOR_VALUE_TYPE") -#define TYPEDEF_VECTOR_VALUE_TYPE(SCALAR_TYPE) \ -typedef SCALAR_TYPE __hip_##SCALAR_TYPE##2_vector_value_type __attribute__((ext_vector_type(2))); \ -typedef SCALAR_TYPE __hip_##SCALAR_TYPE##3_vector_value_type __attribute__((ext_vector_type(3))); \ -typedef SCALAR_TYPE __hip_##SCALAR_TYPE##4_vector_value_type __attribute__((ext_vector_type(4))); \ -typedef SCALAR_TYPE __hip_##SCALAR_TYPE##8_vector_value_type __attribute__((ext_vector_type(8))); \ -typedef SCALAR_TYPE __hip_##SCALAR_TYPE##16_vector_value_type __attribute__((ext_vector_type(16))); - -TYPEDEF_VECTOR_VALUE_TYPE(float); -TYPEDEF_VECTOR_VALUE_TYPE(int); -TYPEDEF_VECTOR_VALUE_TYPE(uint); - -#undef TYPEDEF_VECTOR_VALUE_TYPE -#pragma pop_macro("TYPEDEF_VECTOR_VALUE_TYPE") - -union TData { - __hip_float4_vector_value_type f; - __hip_int4_vector_value_type i; - __hip_uint4_vector_value_type u; -}; - -#define __TEXTURE_FUNCTIONS_DECL__ static inline __device__ - - -#if __clang__ -#define ADDRESS_SPACE_CONSTANT __attribute__((address_space(4))) -#else -#define ADDRESS_SPACE_CONSTANT __attribute__((address_space(2))) -#endif - -#define TEXTURE_PARAMETERS_INIT \ - unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)textureObject; \ - unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; \ - TData texel; -#define TEXTURE_REF_PARAMETERS_INIT \ - unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)texRef.textureObject; \ - unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; \ - TData texel; -#define TEXTURE_SET_FLOAT *retVal = texel.f.x; - -#define TEXTURE_SET_SIGNED *retVal = texel.i.x; - -#define TEXTURE_SET_UNSIGNED *retVal = texel.u.x; - -#define TEXTURE_SET_FLOAT_X retVal->x = texel.f.x; - -#define TEXTURE_SET_SIGNED_X retVal->x = texel.i.x; - -#define TEXTURE_SET_UNSIGNED_X retVal->x = texel.u.x; - -#define TEXTURE_SET_FLOAT_XY \ - retVal->x = texel.f.x; \ - retVal->y = texel.f.y; - -#define TEXTURE_SET_SIGNED_XY \ - retVal->x = texel.i.x; \ - retVal->y = texel.i.y; - -#define TEXTURE_SET_UNSIGNED_XY \ - retVal->x = texel.u.x; \ - retVal->y = texel.u.y; - -#define TEXTURE_SET_FLOAT_XYZW \ - retVal->x = texel.f.x; \ - retVal->y = texel.f.y; \ - retVal->z = texel.f.z; \ - retVal->w = texel.f.w; - -#define TEXTURE_SET_SIGNED_XYZW \ - retVal->x = texel.i.x; \ - retVal->y = texel.i.y; \ - retVal->z = texel.i.z; \ - retVal->w = texel.i.w; - -#define TEXTURE_SET_UNSIGNED_XYZW \ - retVal->x = texel.u.x; \ - retVal->y = texel.u.y; \ - retVal->z = texel.u.z; \ - retVal->w = texel.u.w; - -#define TEXTURE_RETURN_CHAR return texel.i.x; - -#define TEXTURE_RETURN_UCHAR return texel.u.x; - -#define TEXTURE_RETURN_SHORT return texel.i.x; - -#define TEXTURE_RETURN_USHORT return texel.u.x; - -#define TEXTURE_RETURN_INT return texel.i.x; - -#define TEXTURE_RETURN_UINT return texel.u.x; - -#define TEXTURE_RETURN_SIGNED return texel.i.x; - -#define TEXTURE_RETURN_UNSIGNED return texel.u.x; - -#define TEXTURE_RETURN_CHAR_X return make_char1(texel.i.x); - -#define TEXTURE_RETURN_UCHAR_X return make_uchar1(texel.u.x); - -#define TEXTURE_RETURN_SHORT_X return make_short1(texel.i.x); - -#define TEXTURE_RETURN_USHORT_X return make_ushort1(texel.u.x); - -#define TEXTURE_RETURN_INT_X return make_int1(texel.i.x); - -#define TEXTURE_RETURN_UINT_X return make_uint1(texel.u.x); - -#define TEXTURE_RETURN_CHAR_XY return make_char2(texel.i.x, texel.i.y); - -#define TEXTURE_RETURN_UCHAR_XY return make_uchar2(texel.u.x, texel.u.y); - -#define TEXTURE_RETURN_SHORT_XY return make_short2(texel.i.x, texel.i.y); - -#define TEXTURE_RETURN_USHORT_XY return make_ushort2(texel.u.x, texel.u.y); - -#define TEXTURE_RETURN_INT_XY return make_int2(texel.i.x, texel.i.y); - -#define TEXTURE_RETURN_UINT_XY return make_uint2(texel.u.x, texel.u.y); - -#define TEXTURE_RETURN_CHAR_XYZW return make_char4(texel.i.x, texel.i.y, texel.i.z, texel.i.w); - -#define TEXTURE_RETURN_UCHAR_XYZW return make_uchar4(texel.u.x, texel.u.y, texel.u.z, texel.u.w); - -#define TEXTURE_RETURN_SHORT_XYZW return make_short4(texel.i.x, texel.i.y, texel.i.z, texel.i.w); - -#define TEXTURE_RETURN_USHORT_XYZW return make_ushort4(texel.u.x, texel.u.y, texel.u.z, texel.u.w); - -#define TEXTURE_RETURN_INT_XYZW return make_int4(texel.i.x, texel.i.y, texel.i.z, texel.i.w); - -#define TEXTURE_RETURN_UINT_XYZW return make_uint4(texel.u.x, texel.u.y, texel.u.z, texel.u.w); - -#define TEXTURE_RETURN_FLOAT return texel.f.x; - -#define TEXTURE_RETURN_FLOAT_X return make_float1(texel.f.x); - -#define TEXTURE_RETURN_FLOAT_XY return make_float2(texel.f.x, texel.f.y); - -#define TEXTURE_RETURN_FLOAT_XYZW return make_float4(texel.f.x, texel.f.y, texel.f.z, texel.f.w); - -extern "C" { - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_1D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - float c); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_1Da( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_2D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c); - - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_2Da( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c); - -__device__ -float __ockl_image_sample_2Dad( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c); - -__device__ -float __ockl_image_sample_2Dd( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_3D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_grad_1D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - float c, float dx, float dy); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_grad_1Da( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c, float dx, float dy); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_grad_2D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c, __hip_float2_vector_value_type dx, __hip_float2_vector_value_type dy); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_grad_2Da( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c, __hip_float2_vector_value_type dx, __hip_float2_vector_value_type dy); - -__device__ -float __ockl_image_sample_grad_2Dad( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c, __hip_float2_vector_value_type dx, __hip_float2_vector_value_type dy); - -__device__ -float __ockl_image_sample_grad_2Dd( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c, __hip_float2_vector_value_type dx, __hip_float2_vector_value_type dy); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_grad_3D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c, __hip_float4_vector_value_type dx, __hip_float4_vector_value_type dy); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_lod_1D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - float c, float l); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_lod_1Da( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c, float l); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_lod_2D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c, float l); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_lod_2Da( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c, float l); - -__device__ -float __ockl_image_sample_lod_2Dad( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c, float l); - -__device__ -float __ockl_image_sample_lod_2Dd( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float2_vector_value_type c, float l); - -__device__ -__hip_float4_vector_value_type __ockl_image_sample_lod_3D( - unsigned int ADDRESS_SPACE_CONSTANT* i, unsigned int ADDRESS_SPACE_CONSTANT* s, - __hip_float4_vector_value_type c, float l); -} - -//////////////////////////////////////////////////////////// -// Texture object APIs -//////////////////////////////////////////////////////////// - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(char* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(char1* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(char2* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(char4* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(unsigned char* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(uchar1* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(uchar2* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(uchar4* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(short* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(short1* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(short2* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(short4* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(unsigned short* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(ushort1* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(ushort2* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(ushort4* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(int* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(int1* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(int2* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(int4* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(unsigned int* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(uint1* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(uint2* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(uint4* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(float* retVal, hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(float1* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(float2* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1Dfetch(float4* retVal, hipTextureObject_t textureObject, - int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex1Dfetch(hipTextureObject_t textureObject, int x) { - T ret; - tex1Dfetch(&ret, textureObject, x); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex1D(char* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(char1* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(char2* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(char4* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(unsigned char* retVal, hipTextureObject_t textureObject, - float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(uchar1* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(uchar2* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(uchar4* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(short* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(short1* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(short2* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(short4* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(unsigned short* retVal, hipTextureObject_t textureObject, - float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(ushort1* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(ushort2* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(ushort4* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(int* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(int1* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(int2* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(int4* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(unsigned int* retVal, hipTextureObject_t textureObject, - float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(uint1* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(uint2* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(uint4* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(float* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(float1* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(float2* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1D(float4* retVal, hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_SET_FLOAT_XYZW; -} -template -__TEXTURE_FUNCTIONS_DECL__ T tex1D(hipTextureObject_t textureObject, float x) { - T ret; - tex1D(&ret, textureObject, x); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(char* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(char1* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(char2* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(char4* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(unsigned char* retVal, hipTextureObject_t textureObject, - float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(uchar1* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(uchar2* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(uchar4* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(short* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(short1* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(short2* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(short4* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(unsigned short* retVal, hipTextureObject_t textureObject, - float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(ushort1* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(ushort2* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(ushort4* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(int* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(int1* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(int2* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(int4* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(unsigned int* retVal, hipTextureObject_t textureObject, - float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(uint1* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(uint2* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(uint4* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(float* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(float1* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(float2* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLod(float4* retVal, hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex1DLod(hipTextureObject_t textureObject, float x, float level) { - T ret; - tex1DLod(&ret, textureObject, x, level); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(char* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(char1* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(char2* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(char4* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(unsigned char* retVal, hipTextureObject_t textureObject, - float x, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(uchar1* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(uchar2* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(uchar4* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(short* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(short1* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(short2* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(short4* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(unsigned short* retVal, hipTextureObject_t textureObject, - float x, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(ushort1* retVal, hipTextureObject_t textureObject, - float x, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(ushort2* retVal, hipTextureObject_t textureObject, - float x, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(ushort4* retVal, hipTextureObject_t textureObject, - float x, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(int* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(int1* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(int2* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(int4* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(unsigned int* retVal, hipTextureObject_t textureObject, - float x, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(uint1* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(uint2* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(uint4* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(float* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(float1* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(float2* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DGrad(float4* retVal, hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex1DGrad(hipTextureObject_t textureObject, float x, float dx, - float dy) { - T ret; - tex1DLod(&ret, textureObject, x, dx, dy); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex2D(char* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(char1* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(char2* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(char4* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(unsigned char* retVal, hipTextureObject_t textureObject, - float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(uchar1* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(uchar2* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(uchar4* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(short* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(short1* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(short2* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(short4* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(unsigned short* retVal, hipTextureObject_t textureObject, - float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(ushort1* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(ushort2* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(ushort4* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(int* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(int1* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(int2* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(int4* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(unsigned int* retVal, hipTextureObject_t textureObject, - float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(uint1* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(uint2* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(uint4* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(float* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(float1* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(float2* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2D(float4* retVal, hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex2D(hipTextureObject_t textureObject, float x, float y) { - T ret; - tex2D(&ret, textureObject, x, y); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(char* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(char1* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(char2* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(char4* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(unsigned char* retVal, hipTextureObject_t textureObject, - float x, float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(uchar1* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(uchar2* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(uchar4* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(short* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(short1* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(short2* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(short4* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(unsigned short* retVal, hipTextureObject_t textureObject, - float x, float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(ushort1* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(ushort2* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(ushort4* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(int* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(int1* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(int2* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(int4* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(unsigned int* retVal, hipTextureObject_t textureObject, - float x, float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(uint1* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(uint2* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(uint4* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(float* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(float1* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(float2* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLod(float4* retVal, hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex2DLod(hipTextureObject_t textureObject, float x, float y, - float level) { - T ret; - tex2DLod(&ret, textureObject, x, y, level); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex3D(char* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(char1* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(char2* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(char4* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(unsigned char* retVal, hipTextureObject_t textureObject, - float x, float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(uchar1* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(uchar2* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(uchar4* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(short* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(short1* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(short2* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(short4* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(unsigned short* retVal, hipTextureObject_t textureObject, - float x, float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(ushort1* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(ushort2* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(ushort4* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(int* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(int1* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(int2* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(int4* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(unsigned int* retVal, hipTextureObject_t textureObject, - float x, float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(uint1* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(uint2* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(uint4* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(float* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(float1* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(float2* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3D(float4* retVal, hipTextureObject_t textureObject, float x, - float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex3D(hipTextureObject_t textureObject, float x, float y, float z) { - T ret; - tex3D(&ret, textureObject, x, y, z); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(char* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(char1* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(char2* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(char4* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(unsigned char* retVal, hipTextureObject_t textureObject, - float x, float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(uchar1* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(uchar2* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(uchar4* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(short* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(short1* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(short2* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(short4* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(unsigned short* retVal, hipTextureObject_t textureObject, - float x, float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(ushort1* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(ushort2* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(ushort4* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(int* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(int1* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(int2* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(int4* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(unsigned int* retVal, hipTextureObject_t textureObject, - float x, float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(uint1* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(uint2* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(uint4* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(float* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(float1* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(float2* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex3DLod(float4* retVal, hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex3DLod(hipTextureObject_t textureObject, float x, float y, float z, - float level) { - T ret; - tex3DLod(&ret, textureObject, x, y, z, level); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(char* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(char1* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(char2* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_XY; -} -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(char4* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(unsigned char* retVal, - hipTextureObject_t textureObject, float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(uchar1* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(uchar2* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_XY; -} -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(uchar4* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(short* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(short1* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(short2* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_XY; -} -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(short4* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(unsigned short* retVal, - hipTextureObject_t textureObject, float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(ushort1* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(ushort2* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_XY; -} -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(ushort4* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(int* retVal, hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(int1* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(int2* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_XY; -} -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(int4* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(unsigned int* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(uint1* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(uint2* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_XY; -} -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(uint4* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(float* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(float1* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(float2* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_FLOAT_XY; -} -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayered(float4* retVal, hipTextureObject_t textureObject, - float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex1DLayered(hipTextureObject_t textureObject, float x, int layer) { - T ret; - tex1DLayered(&ret, textureObject, x, layer); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(char* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(char1* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(char2* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(char4* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(unsigned char* retVal, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(uchar1* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(uchar2* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(uchar4* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(short* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(short1* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(short2* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(short4* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(unsigned short* retVal, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(ushort1* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(ushort2* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(ushort4* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(int* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(int1* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(int2* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(int4* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(unsigned int* retVal, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(uint1* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(uint2* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(uint4* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(float* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(float1* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(float2* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredLod(float4* retVal, hipTextureObject_t textureObject, - float x, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex1DLayeredLod(hipTextureObject_t textureObject, float x, int layer, - float level) { - T ret; - tex1DLayeredLod(&ret, textureObject, x, layer, level); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(char* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(char1* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(char2* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(char4* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(unsigned char* retVal, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(uchar1* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(uchar2* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(uchar4* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(short* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(short1* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(short2* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(short4* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(unsigned short* retVal, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(ushort1* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(ushort2* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(ushort4* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(int* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(int1* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(int2* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(int4* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(unsigned int* retVal, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(uint1* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(uint2* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(uint4* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(float* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(float1* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(float2* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex1DLayeredGrad(float4* retVal, hipTextureObject_t textureObject, - float x, int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex1DLayeredGrad(hipTextureObject_t textureObject, float x, int layer, - float dx, float dy) { - T ret; - tex1DLayeredGrad(&ret, textureObject, x, layer, dx, dy); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(char* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(char1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(char2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(char4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(unsigned char* retVal, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(uchar1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(uchar2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(uchar4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(short* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(short1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(short2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(short4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(unsigned short* retVal, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(ushort1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(ushort2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(ushort4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(int* retVal, hipTextureObject_t textureObject, float x, - float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(int1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(int2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(int4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(unsigned int* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(uint1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(uint2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(uint4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(float* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(float1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(float2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayered(float4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex2DLayered(hipTextureObject_t textureObject, float x, float y, - int layer) { - T ret; - tex2DLayered(&ret, textureObject, x, y, layer); - return ret; -} - -//////////////////////////////////////////////////////////// -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(char* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(char1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(char2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(char4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(unsigned char* retVal, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(uchar1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(uchar2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(uchar4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(short* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(short1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(short2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(short4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(unsigned short* retVal, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(ushort1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(ushort2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(ushort4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(int* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(int1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(int2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(int4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_SIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(unsigned int* retVal, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(uint1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(uint2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(uint4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_UNSIGNED_XYZW; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(float* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_FLOAT; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(float1* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_FLOAT_X; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(float2* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_FLOAT_XY; -} - -__TEXTURE_FUNCTIONS_DECL__ void tex2DLayeredLod(float4* retVal, hipTextureObject_t textureObject, - float x, float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_SET_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ T tex2DLayeredLod(hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - T ret; - tex2DLayeredLod(&ret, textureObject, x, y, layer, level); - return ret; -} - -//////////////////////////////////////////////////////////// -// Texture Reference APIs -//////////////////////////////////////////////////////////// -template -__TEXTURE_FUNCTIONS_DECL__ char tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1Dfetch(texture texRef, - int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1Dfetch(texture texRef, - int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1Dfetch(texture texRef, - int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1Dfetch(texture texRef, int x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1Dfetch(texture texRef, - hipTextureObject_t textureObject, int x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// -template -__TEXTURE_FUNCTIONS_DECL__ char tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1D(texture texRef, - float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1D(texture texRef, - float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// -template -__TEXTURE_FUNCTIONS_DECL__ char tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT; -} -////// - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1D(texture texRef, float x) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1D(texture texRef, - hipTextureObject_t textureObject, float x) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1D(i, s, x); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_CHAR_X; -} -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DLod(texture texRef, - float x, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DLod(texture texRef, - float x, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DLod(texture texRef, float x, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DLod(texture texRef, - float x, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DLod(texture texRef, float x, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_CHAR_X; -} -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DLod(texture texRef, - hipTextureObject_t textureObject, float x, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_1D(i, s, x, level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DGrad(texture texRef, - float x, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DGrad(texture texRef, - float x, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DGrad(texture texRef, - float x, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DGrad(texture texRef, float x, float dx, - float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DGrad(texture texRef, float x, - float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float dx, - float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_1D(i, s, x, dx, dy); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2D(texture texRef, - float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2D(texture texRef, - float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2D(texture texRef, float x, - float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UINT_XYZW; -} - - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2D(texture texRef, - hipTextureObject_t textureObject, float x, - float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2D(texture texRef, float x, float y) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2D(texture texRef, - hipTextureObject_t textureObject, float x, float y) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_2D(i, s, float2(x, y).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DLod(texture texRef, - float x, float y, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DLod(texture texRef, - float x, float y, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DLod(texture texRef, float x, - float y, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DLod(texture texRef, float x, - float y, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DLod(texture texRef, float x, - float y, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DLod(texture texRef, - float x, float y, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DLod(texture texRef, float x, float y, - float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DGrad(texture texRef, - float x, float y, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DGrad(texture texRef, - float x, float y, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DGrad(texture texRef, float x, - float y, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DGrad(texture texRef, float x, - float y, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DGrad(texture texRef, float x, - float y, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DGrad(texture texRef, - float x, float y, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DGrad(texture texRef, float x, float y, - float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex3D(texture texRef, - float x, float y, float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex3D(texture texRef, - float x, float y, float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex3D(texture texRef, float x, - float y, float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex3D(texture texRef, float x, float y, - float z) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex3D(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex3DLod(texture texRef, - float x, float y, float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex3DLod(texture texRef, - float x, float y, float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex3DLod(texture texRef, float x, float y, - float z, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, float z, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex3DLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, - level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex3DGrad(texture texRef, - float x, float y, float z, float4 dx, - float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex3DGrad(texture texRef, - float x, float y, float z, float4 dx, - float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex3DGrad(texture texRef, float x, - float y, float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex3DGrad(texture texRef, float x, - float y, float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex3DGrad(texture texRef, float x, - float y, float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex3DGrad(texture texRef, - float x, float y, float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex3DGrad(texture texRef, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// -template -__TEXTURE_FUNCTIONS_DECL__ char tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex3DGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - float z, float4 dx, float4 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, - float4(dx.x, dx.y, dx.z, dx.w).data, - float4(dy.x, dy.y, dy.z, dy.w).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DLayered(texture texRef, - float x, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DLayered( - texture texRef, float x, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DLayered(texture texRef, - float x, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DLayered(texture texRef, float x, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DLayered( - texture texRef, hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DLayeredLod( - texture texRef, float x, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DLayeredLod( - texture texRef, float x, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DLayeredLod(texture texRef, - float x, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DLayeredLod(texture texRef, float x, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DLayeredLod( - texture texRef, hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DLayeredLod( - texture texRef, hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, int layer, - float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DLayeredGrad( - texture texRef, float x, int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex1DLayeredGrad( - texture texRef, hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DLayeredGrad( - texture texRef, float x, int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex1DLayeredGrad( - texture texRef, hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DLayeredGrad( - texture texRef, float x, int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex1DLayeredGrad( - texture texRef, hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DLayeredGrad(texture texRef, float x, - int layer, float dx, float dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex1DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - int layer, float dx, float dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dx, dy); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DLayered(texture texRef, float x, float y, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DLayered(texture texRef, - float x, float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DLayered( - texture texRef, float x, float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DLayered( - texture texRef, hipTextureObject_t textureObject, float x, - float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DLayered(texture texRef, float x, float y, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DLayered(texture texRef, float x, float y, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DLayered(texture texRef, float x, float y, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DLayered(texture texRef, float x, float y, - int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DLayered(texture texRef, - float x, float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DLayered(texture texRef, float x, - float y, int layer) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DLayered(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DLayeredLod( - texture texRef, float x, float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DLayeredLod( - texture texRef, hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DLayeredLod( - texture texRef, float x, float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DLayeredLod( - texture texRef, hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DLayeredLod(texture texRef, float x, float y, - int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DLayeredLod(texture texRef, - float x, float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DLayeredLod(texture texRef, float x, - float y, int layer, float level) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DLayeredLod(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float level) { - TEXTURE_PARAMETERS_INIT; - texel.f = __ockl_image_sample_lod_2Da( - i, s, float4(x, y, layer, 0.0f).data, level); - TEXTURE_RETURN_FLOAT_XYZW; -} - -//////////////////////////////////////////////////////////// - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char1 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char2 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ char4 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_CHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DLayeredGrad( - texture texRef, float x, float y, int layer, float2 dx, - float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned char tex2DLayeredGrad( - texture texRef, hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar1 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar2 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uchar4 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UCHAR_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short1 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short2 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ short4 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_SHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DLayeredGrad( - texture texRef, float x, float y, int layer, float2 dx, - float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned short tex2DLayeredGrad( - texture texRef, hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort1 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort2 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ ushort4 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_USHORT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int1 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int2 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ int4 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, float y, - int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_INT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DLayeredGrad( - texture texRef, float x, float y, int layer, float2 dx, - float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ unsigned int tex2DLayeredGrad( - texture texRef, hipTextureObject_t textureObject, float x, float y, - int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint1 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint2 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ uint4 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_UINT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float1 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_X; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float2 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_XY; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DLayeredGrad(texture texRef, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_REF_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_XYZW; -} - -template -__TEXTURE_FUNCTIONS_DECL__ float4 tex2DLayeredGrad(texture texRef, - hipTextureObject_t textureObject, float x, - float y, int layer, float2 dx, float2 dy) { - TEXTURE_PARAMETERS_INIT; - texel.f = - __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, - float2(dx.x, dx.y).data, - float2(dy.x, dy.y).data); - TEXTURE_RETURN_FLOAT_XYZW; -} -#endif diff --git a/src/hipamd/include/hip/amd_detail/texture_indirect_functions.h b/src/hipamd/include/hip/amd_detail/texture_indirect_functions.h deleted file mode 100644 index 87279da8c0..0000000000 --- a/src/hipamd/include/hip/amd_detail/texture_indirect_functions.h +++ /dev/null @@ -1,503 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#if defined(__cplusplus) - -#include -#include -#include - -#if !defined(__HIPCC_RTC__) -#include -#endif // !defined(__HIPCC_RTC__) - -#define TEXTURE_OBJECT_PARAMETERS_INIT \ - unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)textureObject; \ - unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; - -template -struct __hip_is_itex_channel_type -{ - static constexpr bool value = - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value; -}; - -template< - typename T, - unsigned int rank> -struct __hip_is_itex_channel_type> -{ - static constexpr bool value = - __hip_is_itex_channel_type::value && - ((rank == 1) || - (rank == 2) || - (rank == 4)); -}; - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex1Dfetch(hipTextureObject_t textureObject, int x) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_load_1Db(i, x); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex1Dfetch(T *ptr, hipTextureObject_t textureObject, int x) -{ - *ptr = tex1Dfetch(textureObject, x); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex1D(hipTextureObject_t textureObject, float x) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_1D(i, s, x); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex1D(T *ptr, hipTextureObject_t textureObject, float x) -{ - *ptr = tex1D(textureObject, x); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex2D(hipTextureObject_t textureObject, float x, float y) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_2D(i, s, float2(x, y).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex2D(T *ptr, hipTextureObject_t textureObject, float x, float y) -{ - *ptr = tex2D(textureObject, x, y); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex3D(hipTextureObject_t textureObject, float x, float y, float z) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex3D(T *ptr, hipTextureObject_t textureObject, float x, float y, float z) -{ - *ptr = tex3D(textureObject, x, y, z); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex1DLayered(hipTextureObject_t textureObject, float x, int layer) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex1DLayered(T *ptr, hipTextureObject_t textureObject, float x, int layer) -{ - *ptr = tex1DLayered(textureObject, x, layer); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex2DLayered(hipTextureObject_t textureObject, float x, float y, int layer) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex2DLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer) -{ - *ptr = tex1DLayered(textureObject, x, y, layer); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T texCubemap(hipTextureObject_t textureObject, float x, float y, float z) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_CM(i, s, float4(x, y, z, 0.0f).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void texCubemap(T *ptr, hipTextureObject_t textureObject, float x, float y, float z) -{ - *ptr = texCubemap(textureObject, x, y, z); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T texCubemapLayered(hipTextureObject_t textureObject, float x, float y, float z, int layer) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_CMa(i, s, float4(x, y, z, layer).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void texCubemapLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer) -{ - *ptr = texCubemapLayered(textureObject, x, y, z, layer); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex2Dgather(hipTextureObject_t textureObject, float x, float y, int comp = 0) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - switch (comp) { - case 1: { - auto tmp = __ockl_image_gather4r_2D(i, s, float2(x, y).data); - return *reinterpret_cast(&tmp); - break; - } - case 2: { - auto tmp = __ockl_image_gather4g_2D(i, s, float2(x, y).data); - return *reinterpret_cast(&tmp); - break; - } - case 3: { - auto tmp = __ockl_image_gather4b_2D(i, s, float2(x, y).data); - return *reinterpret_cast(&tmp); - break; - } - default: { - auto tmp = __ockl_image_gather4a_2D(i, s, float2(x, y).data); - return *reinterpret_cast(&tmp); - break; - } - }; - return {}; -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex2Dgather(T *ptr, hipTextureObject_t textureObject, float x, float y, int comp = 0) -{ - *ptr = texCubemapLayered(textureObject, x, y, comp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex1DLod(hipTextureObject_t textureObject, float x, float level) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_lod_1D(i, s, x, level); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex1DLod(T *ptr, hipTextureObject_t textureObject, float x, float level) -{ - *ptr = tex1DLod(textureObject, x, level); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex2DLod(hipTextureObject_t textureObject, float x, float y, float level) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex2DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float level) -{ - *ptr = tex2DLod(textureObject, x, y, level); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex3DLod(hipTextureObject_t textureObject, float x, float y, float z, float level) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, level); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex3DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level) -{ - *ptr = tex3DLod(textureObject, x, y, z, level); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex1DLayeredLod(hipTextureObject_t textureObject, float x, int layer, float level) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex1DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, int layer, float level) -{ - *ptr = tex1DLayeredLod(textureObject, x, layer, level); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex2DLayeredLod(hipTextureObject_t textureObject, float x, float y, int layer, float level) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex2DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float level) -{ - *ptr = tex2DLayeredLod(textureObject, x, y, layer, level); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T texCubemapLod(hipTextureObject_t textureObject, float x, float y, float z, float level) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_lod_CM(i, s, float4(x, y, z, 0.0f).data, level); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void texCubemapLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level) -{ - *ptr = texCubemapLod(textureObject, x, y, z, level); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T texCubemapGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - // TODO missing in device libs. - // auto tmp = __ockl_image_sample_grad_CM(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); - // return *reinterpret_cast(&tmp); - return {}; -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void texCubemapGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) -{ - *ptr = texCubemapGrad(textureObject, x, y, z, dPdx, dPdy); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T texCubemapLayeredLod(hipTextureObject_t textureObject, float x, float y, float z, int layer, float level) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_lod_CMa(i, s, float4(x, y, z, layer).data, level); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void texCubemapLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float level) -{ - *ptr = texCubemapLayeredLod(textureObject, x, y, z, layer, level); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex1DGrad(hipTextureObject_t textureObject, float x, float dPdx, float dPdy) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex1DGrad(T *ptr, hipTextureObject_t textureObject, float x, float dPdx, float dPdy) -{ - *ptr = tex1DGrad(textureObject, x, dPdx, dPdy); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex2DGrad(hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex2DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy) -{ - *ptr = tex2DGrad(textureObject, x, y, dPdx, dPdy); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex3DGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex3DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) -{ - *ptr = tex3DGrad(textureObject, x, y, z, dPdx, dPdy); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex1DLayeredGrad(hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dPdx, dPdy); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex1DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy) -{ - *ptr = tex1DLayeredGrad(textureObject, x, layer, dPdx, dPdy); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T tex2DLayeredGrad(hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); - return *reinterpret_cast(&tmp); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void tex2DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy) -{ - *ptr = tex2DLayeredGrad(textureObject, x, y, layer, dPdx, dPdy); -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ T texCubemapLayeredGrad(hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy) -{ - TEXTURE_OBJECT_PARAMETERS_INIT - // TODO missing in device libs. - // auto tmp = __ockl_image_sample_grad_CMa(i, s, float4(x, y, z, layer).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); - // return *reinterpret_cast(&tmp); - return {}; -} - -template < - typename T, - typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> -static __device__ void texCubemapLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy) -{ - *ptr = texCubemapLayeredGrad(textureObject, x, y, z, layer, dPdx, dPdy); -} - -#endif diff --git a/src/hipamd/include/hip/hcc_detail b/src/hipamd/include/hip/hcc_detail deleted file mode 120000 index 4931d48978..0000000000 --- a/src/hipamd/include/hip/hcc_detail +++ /dev/null @@ -1 +0,0 @@ -amd_detail \ No newline at end of file diff --git a/src/hipamd/include/hip/nvcc_detail b/src/hipamd/include/hip/nvcc_detail deleted file mode 120000 index e02ee85e4c..0000000000 --- a/src/hipamd/include/hip/nvcc_detail +++ /dev/null @@ -1 +0,0 @@ -nvidia_detail \ No newline at end of file diff --git a/src/hipamd/include/hip/nvidia_detail/nvidia_channel_descriptor.h b/src/hipamd/include/hip/nvidia_detail/nvidia_channel_descriptor.h deleted file mode 100644 index 7eb0e65fda..0000000000 --- a/src/hipamd/include/hip/nvidia_detail/nvidia_channel_descriptor.h +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_CHANNEL_DESCRIPTOR_H -#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_CHANNEL_DESCRIPTOR_H - -#include "channel_descriptor.h" - -#endif diff --git a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_complex.h b/src/hipamd/include/hip/nvidia_detail/nvidia_hip_complex.h deleted file mode 100644 index 10a53d1743..0000000000 --- a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_complex.h +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COMPLEX_H -#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COMPLEX_H - -#include "cuComplex.h" - -typedef cuFloatComplex hipFloatComplex; - -__device__ __host__ static inline float hipCrealf(hipFloatComplex z) { return cuCrealf(z); } - -__device__ __host__ static inline float hipCimagf(hipFloatComplex z) { return cuCimagf(z); } - -__device__ __host__ static inline hipFloatComplex make_hipFloatComplex(float a, float b) { - return make_cuFloatComplex(a, b); -} - -__device__ __host__ static inline hipFloatComplex hipConjf(hipFloatComplex z) { return cuConjf(z); } - -__device__ __host__ static inline float hipCsqabsf(hipFloatComplex z) { - return cuCabsf(z) * cuCabsf(z); -} - -__device__ __host__ static inline hipFloatComplex hipCaddf(hipFloatComplex p, hipFloatComplex q) { - return cuCaddf(p, q); -} - -__device__ __host__ static inline hipFloatComplex hipCsubf(hipFloatComplex p, hipFloatComplex q) { - return cuCsubf(p, q); -} - -__device__ __host__ static inline hipFloatComplex hipCmulf(hipFloatComplex p, hipFloatComplex q) { - return cuCmulf(p, q); -} - -__device__ __host__ static inline hipFloatComplex hipCdivf(hipFloatComplex p, hipFloatComplex q) { - return cuCdivf(p, q); -} - -__device__ __host__ static inline float hipCabsf(hipFloatComplex z) { return cuCabsf(z); } - -typedef cuDoubleComplex hipDoubleComplex; - -__device__ __host__ static inline double hipCreal(hipDoubleComplex z) { return cuCreal(z); } - -__device__ __host__ static inline double hipCimag(hipDoubleComplex z) { return cuCimag(z); } - -__device__ __host__ static inline hipDoubleComplex make_hipDoubleComplex(double a, double b) { - return make_cuDoubleComplex(a, b); -} - -__device__ __host__ static inline hipDoubleComplex hipConj(hipDoubleComplex z) { return cuConj(z); } - -__device__ __host__ static inline double hipCsqabs(hipDoubleComplex z) { - return cuCabs(z) * cuCabs(z); -} - -__device__ __host__ static inline hipDoubleComplex hipCadd(hipDoubleComplex p, hipDoubleComplex q) { - return cuCadd(p, q); -} - -__device__ __host__ static inline hipDoubleComplex hipCsub(hipDoubleComplex p, hipDoubleComplex q) { - return cuCsub(p, q); -} - -__device__ __host__ static inline hipDoubleComplex hipCmul(hipDoubleComplex p, hipDoubleComplex q) { - return cuCmul(p, q); -} - -__device__ __host__ static inline hipDoubleComplex hipCdiv(hipDoubleComplex p, hipDoubleComplex q) { - return cuCdiv(p, q); -} - -__device__ __host__ static inline double hipCabs(hipDoubleComplex z) { return cuCabs(z); } - -typedef cuFloatComplex hipComplex; - -__device__ __host__ static inline hipComplex make_Complex(float x, float y) { - return make_cuComplex(x, y); -} - -__device__ __host__ static inline hipFloatComplex hipComplexDoubleToFloat(hipDoubleComplex z) { - return cuComplexDoubleToFloat(z); -} - -__device__ __host__ static inline hipDoubleComplex hipComplexFloatToDouble(hipFloatComplex z) { - return cuComplexFloatToDouble(z); -} - -__device__ __host__ static inline hipComplex hipCfmaf(hipComplex p, hipComplex q, hipComplex r) { - return cuCfmaf(p, q, r); -} - -__device__ __host__ static inline hipDoubleComplex hipCfma(hipDoubleComplex p, hipDoubleComplex q, - hipDoubleComplex r) { - return cuCfma(p, q, r); -} - -#endif diff --git a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_cooperative_groups.h b/src/hipamd/include/hip/nvidia_detail/nvidia_hip_cooperative_groups.h deleted file mode 100644 index fc98ae2281..0000000000 --- a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_cooperative_groups.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COOPERATIVE_GROUPS_H -#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COOPERATIVE_GROUPS_H - -// Include CUDA headers -#include -#include - -// Include HIP wrapper headers around CUDA -#include -#include - -#endif // HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COOPERATIVE_GROUPS_H diff --git a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime.h b/src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime.h deleted file mode 100644 index a42fecc611..0000000000 --- a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime.h +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_RUNTIME_H -#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_RUNTIME_H - -#include - -#include - -#define HIP_KERNEL_NAME(...) __VA_ARGS__ - -typedef int hipLaunchParm; - -#define hipLaunchKernelGGLInternal(kernelName, numBlocks, numThreads, memPerBlock, streamId, ...) \ - do { \ - kernelName<<>>(__VA_ARGS__); \ - } while (0) - -#define hipLaunchKernelGGL(kernelName, ...) hipLaunchKernelGGLInternal((kernelName), __VA_ARGS__) - -#define hipReadModeElementType cudaReadModeElementType - -#ifdef __CUDA_ARCH__ - - -// 32-bit Atomics: -#define __HIP_ARCH_HAS_GLOBAL_INT32_ATOMICS__ (__CUDA_ARCH__ >= 110) -#define __HIP_ARCH_HAS_GLOBAL_FLOAT_ATOMIC_EXCH__ (__CUDA_ARCH__ >= 110) -#define __HIP_ARCH_HAS_SHARED_INT32_ATOMICS__ (__CUDA_ARCH__ >= 120) -#define __HIP_ARCH_HAS_SHARED_FLOAT_ATOMIC_EXCH__ (__CUDA_ARCH__ >= 120) -#define __HIP_ARCH_HAS_FLOAT_ATOMIC_ADD__ (__CUDA_ARCH__ >= 200) - -// 64-bit Atomics: -#define __HIP_ARCH_HAS_GLOBAL_INT64_ATOMICS__ (__CUDA_ARCH__ >= 200) -#define __HIP_ARCH_HAS_SHARED_INT64_ATOMICS__ (__CUDA_ARCH__ >= 120) - -// Doubles -#define __HIP_ARCH_HAS_DOUBLES__ (__CUDA_ARCH__ >= 120) - -// warp cross-lane operations: -#define __HIP_ARCH_HAS_WARP_VOTE__ (__CUDA_ARCH__ >= 120) -#define __HIP_ARCH_HAS_WARP_BALLOT__ (__CUDA_ARCH__ >= 200) -#define __HIP_ARCH_HAS_WARP_SHUFFLE__ (__CUDA_ARCH__ >= 300) -#define __HIP_ARCH_HAS_WARP_FUNNEL_SHIFT__ (__CUDA_ARCH__ >= 350) - -// sync -#define __HIP_ARCH_HAS_THREAD_FENCE_SYSTEM__ (__CUDA_ARCH__ >= 200) -#define __HIP_ARCH_HAS_SYNC_THREAD_EXT__ (__CUDA_ARCH__ >= 200) - -// misc -#define __HIP_ARCH_HAS_SURFACE_FUNCS__ (__CUDA_ARCH__ >= 200) -#define __HIP_ARCH_HAS_3DGRID__ (__CUDA_ARCH__ >= 200) -#define __HIP_ARCH_HAS_DYNAMIC_PARALLEL__ (__CUDA_ARCH__ >= 350) - -#endif - -#ifdef __CUDACC__ - - -#define hipThreadIdx_x threadIdx.x -#define hipThreadIdx_y threadIdx.y -#define hipThreadIdx_z threadIdx.z - -#define hipBlockIdx_x blockIdx.x -#define hipBlockIdx_y blockIdx.y -#define hipBlockIdx_z blockIdx.z - -#define hipBlockDim_x blockDim.x -#define hipBlockDim_y blockDim.y -#define hipBlockDim_z blockDim.z - -#define hipGridDim_x gridDim.x -#define hipGridDim_y gridDim.y -#define hipGridDim_z gridDim.z - -#define HIP_SYMBOL(X) &X - -/** - * Map HIP_DYNAMIC_SHARED to "extern __shared__" for compatibility with old HIP applications - * To be removed in a future release. - */ -#define HIP_DYNAMIC_SHARED(type, var) extern __shared__ type var[]; -#define HIP_DYNAMIC_SHARED_ATTRIBUTE - -#ifdef __HIP_DEVICE_COMPILE__ -#define abort_() \ - { asm("trap;"); } -#undef assert -#define assert(COND) \ - { \ - if (!COND) { \ - abort_(); \ - } \ - } -#endif - -#define __clock() clock() -#define __clock64() clock64() - -#endif - -#endif diff --git a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime_api.h b/src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime_api.h deleted file mode 100644 index 66e4743abd..0000000000 --- a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime_api.h +++ /dev/null @@ -1,2195 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_RUNTIME_API_H -#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_RUNTIME_API_H - -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -#define __dparm(x) = x -#else -#define __dparm(x) -#endif - -// Add Deprecated Support for CUDA Mapped HIP APIs -#if defined(__DOXYGEN_ONLY__) || defined(HIP_ENABLE_DEPRECATED) -#define __HIP_DEPRECATED -#elif defined(_MSC_VER) -#define __HIP_DEPRECATED __declspec(deprecated) -#elif defined(__GNUC__) -#define __HIP_DEPRECATED __attribute__((deprecated)) -#else -#define __HIP_DEPRECATED -#endif - - -// TODO -move to include/hip_runtime_api.h as a common implementation. -/** - * Memory copy types - * - */ -typedef enum hipMemcpyKind { - hipMemcpyHostToHost, - hipMemcpyHostToDevice, - hipMemcpyDeviceToHost, - hipMemcpyDeviceToDevice, - hipMemcpyDefault -} hipMemcpyKind; - -typedef enum hipMemoryAdvise { - hipMemAdviseSetReadMostly, - hipMemAdviseUnsetReadMostly, - hipMemAdviseSetPreferredLocation, - hipMemAdviseUnsetPreferredLocation, - hipMemAdviseSetAccessedBy, - hipMemAdviseUnsetAccessedBy -} hipMemoryAdvise; - -// hipDataType -#define hipDataType cudaDataType -#define HIP_R_16F CUDA_R_16F -#define HIP_R_32F CUDA_R_32F -#define HIP_R_64F CUDA_R_64F -#define HIP_C_16F CUDA_C_16F -#define HIP_C_32F CUDA_C_32F -#define HIP_C_64F CUDA_C_64F - -// hipLibraryPropertyType -#define hipLibraryPropertyType libraryPropertyType -#define HIP_LIBRARY_MAJOR_VERSION MAJOR_VERSION -#define HIP_LIBRARY_MINOR_VERSION MINOR_VERSION -#define HIP_LIBRARY_PATCH_LEVEL PATCH_LEVEL - -#define HIP_ARRAY_DESCRIPTOR CUDA_ARRAY_DESCRIPTOR -#define HIP_ARRAY3D_DESCRIPTOR CUDA_ARRAY3D_DESCRIPTOR - -//hipArray_Format -#define HIP_AD_FORMAT_UNSIGNED_INT8 CU_AD_FORMAT_UNSIGNED_INT8 -#define HIP_AD_FORMAT_UNSIGNED_INT16 CU_AD_FORMAT_UNSIGNED_INT16 -#define HIP_AD_FORMAT_UNSIGNED_INT32 CU_AD_FORMAT_UNSIGNED_INT32 -#define HIP_AD_FORMAT_SIGNED_INT8 CU_AD_FORMAT_SIGNED_INT8 -#define HIP_AD_FORMAT_SIGNED_INT16 CU_AD_FORMAT_SIGNED_INT16 -#define HIP_AD_FORMAT_SIGNED_INT32 CU_AD_FORMAT_SIGNED_INT32 -#define HIP_AD_FORMAT_HALF CU_AD_FORMAT_HALF -#define HIP_AD_FORMAT_FLOAT CU_AD_FORMAT_FLOAT - -// hipArray_Format -#define hipArray_Format CUarray_format - -inline static CUarray_format hipArray_FormatToCUarray_format( - hipArray_Format format) { - switch (format) { - case HIP_AD_FORMAT_UNSIGNED_INT8: - return CU_AD_FORMAT_UNSIGNED_INT8; - case HIP_AD_FORMAT_UNSIGNED_INT16: - return CU_AD_FORMAT_UNSIGNED_INT16; - case HIP_AD_FORMAT_UNSIGNED_INT32: - return CU_AD_FORMAT_UNSIGNED_INT32; - case HIP_AD_FORMAT_SIGNED_INT8: - return CU_AD_FORMAT_SIGNED_INT8; - case HIP_AD_FORMAT_SIGNED_INT16: - return CU_AD_FORMAT_SIGNED_INT16; - case HIP_AD_FORMAT_SIGNED_INT32: - return CU_AD_FORMAT_SIGNED_INT32; - case HIP_AD_FORMAT_HALF: - return CU_AD_FORMAT_HALF; - case HIP_AD_FORMAT_FLOAT: - return CU_AD_FORMAT_FLOAT; - default: - return CU_AD_FORMAT_UNSIGNED_INT8; - } -} - -#define HIP_TR_ADDRESS_MODE_WRAP CU_TR_ADDRESS_MODE_WRAP -#define HIP_TR_ADDRESS_MODE_CLAMP CU_TR_ADDRESS_MODE_CLAMP -#define HIP_TR_ADDRESS_MODE_MIRROR CU_TR_ADDRESS_MODE_MIRROR -#define HIP_TR_ADDRESS_MODE_BORDER CU_TR_ADDRESS_MODE_BORDER - -// hipAddress_mode -#define hipAddress_mode CUaddress_mode - -inline static CUaddress_mode hipAddress_modeToCUaddress_mode( - hipAddress_mode mode) { - switch (mode) { - case HIP_TR_ADDRESS_MODE_WRAP: - return CU_TR_ADDRESS_MODE_WRAP; - case HIP_TR_ADDRESS_MODE_CLAMP: - return CU_TR_ADDRESS_MODE_CLAMP; - case HIP_TR_ADDRESS_MODE_MIRROR: - return CU_TR_ADDRESS_MODE_MIRROR; - case HIP_TR_ADDRESS_MODE_BORDER: - return CU_TR_ADDRESS_MODE_BORDER; - default: - return CU_TR_ADDRESS_MODE_WRAP; - } -} - -#define HIP_TR_FILTER_MODE_POINT CU_TR_FILTER_MODE_POINT -#define HIP_TR_FILTER_MODE_LINEAR CU_TR_FILTER_MODE_LINEAR - -// hipFilter_mode -#define hipFilter_mode CUfilter_mode - -inline static CUfilter_mode hipFilter_mode_enumToCUfilter_mode( - hipFilter_mode mode) { - switch (mode) { - case HIP_TR_FILTER_MODE_POINT: - return CU_TR_FILTER_MODE_POINT; - case HIP_TR_FILTER_MODE_LINEAR: - return CU_TR_FILTER_MODE_LINEAR; - default: - return CU_TR_FILTER_MODE_POINT; - } -} - -//hipResourcetype -#define HIP_RESOURCE_TYPE_ARRAY CU_RESOURCE_TYPE_ARRAY -#define HIP_RESOURCE_TYPE_MIPMAPPED_ARRAY CU_RESOURCE_TYPE_MIPMAPPED_ARRAY -#define HIP_RESOURCE_TYPE_LINEAR CU_RESOURCE_TYPE_LINEAR -#define HIP_RESOURCE_TYPE_PITCH2D CU_RESOURCE_TYPE_PITCH2D - -// hipResourcetype -#define hipResourcetype CUresourcetype - -inline static CUresourcetype hipResourcetype_enumToCUresourcetype( - hipResourcetype resType) { - switch (resType) { - case HIP_RESOURCE_TYPE_ARRAY: - return CU_RESOURCE_TYPE_ARRAY; - case HIP_RESOURCE_TYPE_MIPMAPPED_ARRAY: - return CU_RESOURCE_TYPE_MIPMAPPED_ARRAY; - case HIP_RESOURCE_TYPE_LINEAR: - return CU_RESOURCE_TYPE_LINEAR; - case HIP_RESOURCE_TYPE_PITCH2D: - return CU_RESOURCE_TYPE_PITCH2D; - default: - return CU_RESOURCE_TYPE_ARRAY; - } -} - -#define hipTexRef CUtexref -#define hiparray CUarray - -// hipTextureAddressMode -typedef enum cudaTextureAddressMode hipTextureAddressMode; -#define hipAddressModeWrap cudaAddressModeWrap -#define hipAddressModeClamp cudaAddressModeClamp -#define hipAddressModeMirror cudaAddressModeMirror -#define hipAddressModeBorder cudaAddressModeBorder - -// hipTextureFilterMode -typedef enum cudaTextureFilterMode hipTextureFilterMode; -#define hipFilterModePoint cudaFilterModePoint -#define hipFilterModeLinear cudaFilterModeLinear - -// hipTextureReadMode -typedef enum cudaTextureReadMode hipTextureReadMode; -#define hipReadModeElementType cudaReadModeElementType -#define hipReadModeNormalizedFloat cudaReadModeNormalizedFloat - -// hipChannelFormatKind -typedef enum cudaChannelFormatKind hipChannelFormatKind; -#define hipChannelFormatKindSigned cudaChannelFormatKindSigned -#define hipChannelFormatKindUnsigned cudaChannelFormatKindUnsigned -#define hipChannelFormatKindFloat cudaChannelFormatKindFloat -#define hipChannelFormatKindNone cudaChannelFormatKindNone - -// hipMemRangeAttribute -typedef enum cudaMemRangeAttribute hipMemRangeAttribute; -#define hipMemRangeAttributeReadMostly cudaMemRangeAttributeReadMostly -#define hipMemRangeAttributePreferredLocation cudaMemRangeAttributePreferredLocation -#define hipMemRangeAttributeAccessedBy cudaMemRangeAttributeAccessedBy -#define hipMemRangeAttributeLastPrefetchLocation cudaMemRangeAttributeLastPrefetchLocation - -#define hipSurfaceBoundaryMode cudaSurfaceBoundaryMode -#define hipBoundaryModeZero cudaBoundaryModeZero -#define hipBoundaryModeTrap cudaBoundaryModeTrap -#define hipBoundaryModeClamp cudaBoundaryModeClamp - -// hipFuncCache -#define hipFuncCachePreferNone cudaFuncCachePreferNone -#define hipFuncCachePreferShared cudaFuncCachePreferShared -#define hipFuncCachePreferL1 cudaFuncCachePreferL1 -#define hipFuncCachePreferEqual cudaFuncCachePreferEqual - -// hipResourceType -#define hipResourceType cudaResourceType -#define hipResourceTypeArray cudaResourceTypeArray -#define hipResourceTypeMipmappedArray cudaResourceTypeMipmappedArray -#define hipResourceTypeLinear cudaResourceTypeLinear -#define hipResourceTypePitch2D cudaResourceTypePitch2D -// -// hipErrorNoDevice. - - -//! Flags that can be used with hipEventCreateWithFlags: -#define hipEventDefault cudaEventDefault -#define hipEventBlockingSync cudaEventBlockingSync -#define hipEventDisableTiming cudaEventDisableTiming -#define hipEventInterprocess cudaEventInterprocess -#define hipEventReleaseToDevice 0 /* no-op on CUDA platform */ -#define hipEventReleaseToSystem 0 /* no-op on CUDA platform */ - - -#define hipHostMallocDefault cudaHostAllocDefault -#define hipHostMallocPortable cudaHostAllocPortable -#define hipHostMallocMapped cudaHostAllocMapped -#define hipHostMallocWriteCombined cudaHostAllocWriteCombined -#define hipHostMallocCoherent 0x0 -#define hipHostMallocNonCoherent 0x0 - -#define hipMemAttachGlobal cudaMemAttachGlobal -#define hipMemAttachHost cudaMemAttachHost -#define hipMemAttachSingle cudaMemAttachSingle - -#define hipHostRegisterDefault cudaHostRegisterDefault -#define hipHostRegisterPortable cudaHostRegisterPortable -#define hipHostRegisterMapped cudaHostRegisterMapped -#define hipHostRegisterIoMemory cudaHostRegisterIoMemory - -#define HIP_LAUNCH_PARAM_BUFFER_POINTER CU_LAUNCH_PARAM_BUFFER_POINTER -#define HIP_LAUNCH_PARAM_BUFFER_SIZE CU_LAUNCH_PARAM_BUFFER_SIZE -#define HIP_LAUNCH_PARAM_END CU_LAUNCH_PARAM_END -#define hipLimitMallocHeapSize cudaLimitMallocHeapSize -#define hipIpcMemLazyEnablePeerAccess cudaIpcMemLazyEnablePeerAccess - -#define hipOccupancyDefault cudaOccupancyDefault - -#define hipCooperativeLaunchMultiDeviceNoPreSync \ - cudaCooperativeLaunchMultiDeviceNoPreSync -#define hipCooperativeLaunchMultiDeviceNoPostSync \ - cudaCooperativeLaunchMultiDeviceNoPostSync - - -// enum CUjit_option redefines -#define hipJitOptionMaxRegisters CU_JIT_MAX_REGISTERS -#define hipJitOptionThreadsPerBlock CU_JIT_THREADS_PER_BLOCK -#define hipJitOptionWallTime CU_JIT_WALL_TIME -#define hipJitOptionInfoLogBuffer CU_JIT_INFO_LOG_BUFFER -#define hipJitOptionInfoLogBufferSizeBytes CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES -#define hipJitOptionErrorLogBuffer CU_JIT_ERROR_LOG_BUFFER -#define hipJitOptionErrorLogBufferSizeBytes CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES -#define hipJitOptionOptimizationLevel CU_JIT_OPTIMIZATION_LEVEL -#define hipJitOptionTargetFromContext CU_JIT_TARGET_FROM_CUCONTEXT -#define hipJitOptionTarget CU_JIT_TARGET -#define hipJitOptionFallbackStrategy CU_JIT_FALLBACK_STRATEGY -#define hipJitOptionGenerateDebugInfo CU_JIT_GENERATE_DEBUG_INFO -#define hipJitOptionLogVerbose CU_JIT_LOG_VERBOSE -#define hipJitOptionGenerateLineInfo CU_JIT_GENERATE_LINE_INFO -#define hipJitOptionCacheMode CU_JIT_CACHE_MODE -#define hipJitOptionSm3xOpt CU_JIT_NEW_SM3X_OPT -#define hipJitOptionFastCompile CU_JIT_FAST_COMPILE -#define hipJitOptionNumOptions CU_JIT_NUM_OPTIONS - -typedef cudaEvent_t hipEvent_t; -typedef cudaStream_t hipStream_t; -typedef cudaIpcEventHandle_t hipIpcEventHandle_t; -typedef cudaIpcMemHandle_t hipIpcMemHandle_t; -typedef enum cudaLimit hipLimit_t; -typedef enum cudaFuncAttribute hipFuncAttribute; -typedef enum cudaFuncCache hipFuncCache_t; -typedef CUcontext hipCtx_t; -typedef enum cudaSharedMemConfig hipSharedMemConfig; -typedef CUfunc_cache hipFuncCache; -typedef CUjit_option hipJitOption; -typedef CUdevice hipDevice_t; -typedef enum cudaDeviceP2PAttr hipDeviceP2PAttr; -#define hipDevP2PAttrPerformanceRank cudaDevP2PAttrPerformanceRank -#define hipDevP2PAttrAccessSupported cudaDevP2PAttrAccessSupported -#define hipDevP2PAttrNativeAtomicSupported cudaDevP2PAttrNativeAtomicSupported -#define hipDevP2PAttrHipArrayAccessSupported cudaDevP2PAttrCudaArrayAccessSupported -#define hipFuncAttributeMaxDynamicSharedMemorySize cudaFuncAttributeMaxDynamicSharedMemorySize -#define hipFuncAttributePreferredSharedMemoryCarveout cudaFuncAttributePreferredSharedMemoryCarveout - -typedef CUmodule hipModule_t; -typedef CUfunction hipFunction_t; -typedef CUdeviceptr hipDeviceptr_t; -typedef struct cudaArray hipArray; -typedef struct cudaArray* hipArray_t; -typedef struct cudaArray* hipArray_const_t; -typedef struct cudaFuncAttributes hipFuncAttributes; -typedef struct cudaLaunchParams hipLaunchParams; -#define hipFunction_attribute CUfunction_attribute -#define hip_Memcpy2D CUDA_MEMCPY2D -#define HIP_MEMCPY3D CUDA_MEMCPY3D -#define hipMemcpy3DParms cudaMemcpy3DParms -#define hipArrayDefault cudaArrayDefault -#define hipArrayLayered cudaArrayLayered -#define hipArraySurfaceLoadStore cudaArraySurfaceLoadStore -#define hipArrayCubemap cudaArrayCubemap -#define hipArrayTextureGather cudaArrayTextureGather - -typedef cudaTextureObject_t hipTextureObject_t; -typedef cudaSurfaceObject_t hipSurfaceObject_t; -#define hipTextureType1D cudaTextureType1D -#define hipTextureType1DLayered cudaTextureType1DLayered -#define hipTextureType2D cudaTextureType2D -#define hipTextureType2DLayered cudaTextureType2DLayered -#define hipTextureType3D cudaTextureType3D - -#define hipDeviceScheduleAuto cudaDeviceScheduleAuto -#define hipDeviceScheduleSpin cudaDeviceScheduleSpin -#define hipDeviceScheduleYield cudaDeviceScheduleYield -#define hipDeviceScheduleBlockingSync cudaDeviceScheduleBlockingSync -#define hipDeviceScheduleMask cudaDeviceScheduleMask -#define hipDeviceMapHost cudaDeviceMapHost -#define hipDeviceLmemResizeToMax cudaDeviceLmemResizeToMax - -#define hipCpuDeviceId cudaCpuDeviceId -#define hipInvalidDeviceId cudaInvalidDeviceId -typedef struct cudaExtent hipExtent; -typedef struct cudaPitchedPtr hipPitchedPtr; -#define make_hipExtent make_cudaExtent -#define make_hipPos make_cudaPos -#define make_hipPitchedPtr make_cudaPitchedPtr -// Flags that can be used with hipStreamCreateWithFlags -#define hipStreamDefault cudaStreamDefault -#define hipStreamNonBlocking cudaStreamNonBlocking - -typedef struct cudaChannelFormatDesc hipChannelFormatDesc; -typedef struct cudaResourceDesc hipResourceDesc; -typedef struct cudaTextureDesc hipTextureDesc; -typedef struct cudaResourceViewDesc hipResourceViewDesc; -// adding code for hipmemSharedConfig -#define hipSharedMemBankSizeDefault cudaSharedMemBankSizeDefault -#define hipSharedMemBankSizeFourByte cudaSharedMemBankSizeFourByte -#define hipSharedMemBankSizeEightByte cudaSharedMemBankSizeEightByte - -//Function Attributes -#define HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK -#define HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES -#define HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES CU_FUNC_ATTRIBUTE_CONST_SIZE_BYTES -#define HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES -#define HIP_FUNC_ATTRIBUTE_NUM_REGS CU_FUNC_ATTRIBUTE_NUM_REGS -#define HIP_FUNC_ATTRIBUTE_PTX_VERSION CU_FUNC_ATTRIBUTE_PTX_VERSION -#define HIP_FUNC_ATTRIBUTE_BINARY_VERSION CU_FUNC_ATTRIBUTE_BINARY_VERSION -#define HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA CU_FUNC_ATTRIBUTE_CACHE_MODE_CA -#define HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES -#define HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT -#define HIP_FUNC_ATTRIBUTE_MAX CU_FUNC_ATTRIBUTE_MAX - -#if CUDA_VERSION >= 9000 -#define __shfl(...) __shfl_sync(0xffffffff, __VA_ARGS__) -#define __shfl_up(...) __shfl_up_sync(0xffffffff, __VA_ARGS__) -#define __shfl_down(...) __shfl_down_sync(0xffffffff, __VA_ARGS__) -#define __shfl_xor(...) __shfl_xor_sync(0xffffffff, __VA_ARGS__) -#endif // CUDA_VERSION >= 9000 - -inline static hipError_t hipCUDAErrorTohipError(cudaError_t cuError) { - switch (cuError) { - case cudaSuccess: - return hipSuccess; - case cudaErrorProfilerDisabled: - return hipErrorProfilerDisabled; - case cudaErrorProfilerNotInitialized: - return hipErrorProfilerNotInitialized; - case cudaErrorProfilerAlreadyStarted: - return hipErrorProfilerAlreadyStarted; - case cudaErrorProfilerAlreadyStopped: - return hipErrorProfilerAlreadyStopped; - case cudaErrorInsufficientDriver: - return hipErrorInsufficientDriver; - case cudaErrorUnsupportedLimit: - return hipErrorUnsupportedLimit; - case cudaErrorPeerAccessUnsupported: - return hipErrorPeerAccessUnsupported; - case cudaErrorInvalidGraphicsContext: - return hipErrorInvalidGraphicsContext; - case cudaErrorSharedObjectSymbolNotFound: - return hipErrorSharedObjectSymbolNotFound; - case cudaErrorSharedObjectInitFailed: - return hipErrorSharedObjectInitFailed; - case cudaErrorOperatingSystem: - return hipErrorOperatingSystem; - case cudaErrorSetOnActiveProcess: - return hipErrorSetOnActiveProcess; - case cudaErrorIllegalAddress: - return hipErrorIllegalAddress; - case cudaErrorInvalidSymbol: - return hipErrorInvalidSymbol; - case cudaErrorMissingConfiguration: - return hipErrorMissingConfiguration; - case cudaErrorMemoryAllocation: - return hipErrorOutOfMemory; - case cudaErrorInitializationError: - return hipErrorNotInitialized; - case cudaErrorLaunchFailure: - return hipErrorLaunchFailure; - case cudaErrorCooperativeLaunchTooLarge: - return hipErrorCooperativeLaunchTooLarge; - case cudaErrorPriorLaunchFailure: - return hipErrorPriorLaunchFailure; - case cudaErrorLaunchOutOfResources: - return hipErrorLaunchOutOfResources; - case cudaErrorInvalidDeviceFunction: - return hipErrorInvalidDeviceFunction; - case cudaErrorInvalidConfiguration: - return hipErrorInvalidConfiguration; - case cudaErrorInvalidDevice: - return hipErrorInvalidDevice; - case cudaErrorInvalidValue: - return hipErrorInvalidValue; - case cudaErrorInvalidDevicePointer: - return hipErrorInvalidDevicePointer; - case cudaErrorInvalidMemcpyDirection: - return hipErrorInvalidMemcpyDirection; - case cudaErrorInvalidResourceHandle: - return hipErrorInvalidHandle; - case cudaErrorNotReady: - return hipErrorNotReady; - case cudaErrorNoDevice: - return hipErrorNoDevice; - case cudaErrorPeerAccessAlreadyEnabled: - return hipErrorPeerAccessAlreadyEnabled; - case cudaErrorPeerAccessNotEnabled: - return hipErrorPeerAccessNotEnabled; - case cudaErrorHostMemoryAlreadyRegistered: - return hipErrorHostMemoryAlreadyRegistered; - case cudaErrorHostMemoryNotRegistered: - return hipErrorHostMemoryNotRegistered; - case cudaErrorMapBufferObjectFailed: - return hipErrorMapFailed; - case cudaErrorAssert: - return hipErrorAssert; - case cudaErrorNotSupported: - return hipErrorNotSupported; - case cudaErrorCudartUnloading: - return hipErrorDeinitialized; - case cudaErrorInvalidKernelImage: - return hipErrorInvalidImage; - case cudaErrorUnmapBufferObjectFailed: - return hipErrorUnmapFailed; - case cudaErrorNoKernelImageForDevice: - return hipErrorNoBinaryForGpu; - case cudaErrorECCUncorrectable: - return hipErrorECCNotCorrectable; - case cudaErrorDeviceAlreadyInUse: - return hipErrorContextAlreadyInUse; - case cudaErrorInvalidPtx: - return hipErrorInvalidKernelFile; - case cudaErrorLaunchTimeout: - return hipErrorLaunchTimeOut; -#if CUDA_VERSION >= 10010 - case cudaErrorInvalidSource: - return hipErrorInvalidSource; - case cudaErrorFileNotFound: - return hipErrorFileNotFound; - case cudaErrorSymbolNotFound: - return hipErrorNotFound; - case cudaErrorArrayIsMapped: - return hipErrorArrayIsMapped; - case cudaErrorNotMappedAsPointer: - return hipErrorNotMappedAsPointer; - case cudaErrorNotMappedAsArray: - return hipErrorNotMappedAsArray; - case cudaErrorNotMapped: - return hipErrorNotMapped; - case cudaErrorAlreadyAcquired: - return hipErrorAlreadyAcquired; - case cudaErrorAlreadyMapped: - return hipErrorAlreadyMapped; -#endif -#if CUDA_VERSION >= 10020 - case cudaErrorDeviceUninitialized: - return hipErrorInvalidContext; -#endif - case cudaErrorUnknown: - default: - return hipErrorUnknown; // Note - translated error. - } -} - -inline static hipError_t hipCUResultTohipError(CUresult cuError) { - switch (cuError) { - case CUDA_SUCCESS: - return hipSuccess; - case CUDA_ERROR_OUT_OF_MEMORY: - return hipErrorOutOfMemory; - case CUDA_ERROR_INVALID_VALUE: - return hipErrorInvalidValue; - case CUDA_ERROR_INVALID_DEVICE: - return hipErrorInvalidDevice; - case CUDA_ERROR_DEINITIALIZED: - return hipErrorDeinitialized; - case CUDA_ERROR_NO_DEVICE: - return hipErrorNoDevice; - case CUDA_ERROR_INVALID_CONTEXT: - return hipErrorInvalidContext; - case CUDA_ERROR_NOT_INITIALIZED: - return hipErrorNotInitialized; - case CUDA_ERROR_INVALID_HANDLE: - return hipErrorInvalidHandle; - case CUDA_ERROR_MAP_FAILED: - return hipErrorMapFailed; - case CUDA_ERROR_PROFILER_DISABLED: - return hipErrorProfilerDisabled; - case CUDA_ERROR_PROFILER_NOT_INITIALIZED: - return hipErrorProfilerNotInitialized; - case CUDA_ERROR_PROFILER_ALREADY_STARTED: - return hipErrorProfilerAlreadyStarted; - case CUDA_ERROR_PROFILER_ALREADY_STOPPED: - return hipErrorProfilerAlreadyStopped; - case CUDA_ERROR_INVALID_IMAGE: - return hipErrorInvalidImage; - case CUDA_ERROR_CONTEXT_ALREADY_CURRENT: - return hipErrorContextAlreadyCurrent; - case CUDA_ERROR_UNMAP_FAILED: - return hipErrorUnmapFailed; - case CUDA_ERROR_ARRAY_IS_MAPPED: - return hipErrorArrayIsMapped; - case CUDA_ERROR_ALREADY_MAPPED: - return hipErrorAlreadyMapped; - case CUDA_ERROR_NO_BINARY_FOR_GPU: - return hipErrorNoBinaryForGpu; - case CUDA_ERROR_ALREADY_ACQUIRED: - return hipErrorAlreadyAcquired; - case CUDA_ERROR_NOT_MAPPED: - return hipErrorNotMapped; - case CUDA_ERROR_NOT_MAPPED_AS_ARRAY: - return hipErrorNotMappedAsArray; - case CUDA_ERROR_NOT_MAPPED_AS_POINTER: - return hipErrorNotMappedAsPointer; - case CUDA_ERROR_ECC_UNCORRECTABLE: - return hipErrorECCNotCorrectable; - case CUDA_ERROR_UNSUPPORTED_LIMIT: - return hipErrorUnsupportedLimit; - case CUDA_ERROR_CONTEXT_ALREADY_IN_USE: - return hipErrorContextAlreadyInUse; - case CUDA_ERROR_PEER_ACCESS_UNSUPPORTED: - return hipErrorPeerAccessUnsupported; - case CUDA_ERROR_INVALID_PTX: - return hipErrorInvalidKernelFile; - case CUDA_ERROR_INVALID_GRAPHICS_CONTEXT: - return hipErrorInvalidGraphicsContext; - case CUDA_ERROR_INVALID_SOURCE: - return hipErrorInvalidSource; - case CUDA_ERROR_FILE_NOT_FOUND: - return hipErrorFileNotFound; - case CUDA_ERROR_SHARED_OBJECT_SYMBOL_NOT_FOUND: - return hipErrorSharedObjectSymbolNotFound; - case CUDA_ERROR_SHARED_OBJECT_INIT_FAILED: - return hipErrorSharedObjectInitFailed; - case CUDA_ERROR_OPERATING_SYSTEM: - return hipErrorOperatingSystem; - case CUDA_ERROR_NOT_FOUND: - return hipErrorNotFound; - case CUDA_ERROR_NOT_READY: - return hipErrorNotReady; - case CUDA_ERROR_ILLEGAL_ADDRESS: - return hipErrorIllegalAddress; - case CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES: - return hipErrorLaunchOutOfResources; - case CUDA_ERROR_LAUNCH_TIMEOUT: - return hipErrorLaunchTimeOut; - case CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED: - return hipErrorPeerAccessAlreadyEnabled; - case CUDA_ERROR_PEER_ACCESS_NOT_ENABLED: - return hipErrorPeerAccessNotEnabled; - case CUDA_ERROR_PRIMARY_CONTEXT_ACTIVE: - return hipErrorSetOnActiveProcess; - case CUDA_ERROR_ASSERT: - return hipErrorAssert; - case CUDA_ERROR_HOST_MEMORY_ALREADY_REGISTERED: - return hipErrorHostMemoryAlreadyRegistered; - case CUDA_ERROR_HOST_MEMORY_NOT_REGISTERED: - return hipErrorHostMemoryNotRegistered; - case CUDA_ERROR_LAUNCH_FAILED: - return hipErrorLaunchFailure; - case CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE: - return hipErrorCooperativeLaunchTooLarge; - case CUDA_ERROR_NOT_SUPPORTED: - return hipErrorNotSupported; - case CUDA_ERROR_UNKNOWN: - default: - return hipErrorUnknown; // Note - translated error. - } -} - -inline static cudaError_t hipErrorToCudaError(hipError_t hError) { - switch (hError) { - case hipSuccess: - return cudaSuccess; - case hipErrorOutOfMemory: - return cudaErrorMemoryAllocation; - case hipErrorProfilerDisabled: - return cudaErrorProfilerDisabled; - case hipErrorProfilerNotInitialized: - return cudaErrorProfilerNotInitialized; - case hipErrorProfilerAlreadyStarted: - return cudaErrorProfilerAlreadyStarted; - case hipErrorProfilerAlreadyStopped: - return cudaErrorProfilerAlreadyStopped; - case hipErrorInvalidConfiguration: - return cudaErrorInvalidConfiguration; - case hipErrorLaunchOutOfResources: - return cudaErrorLaunchOutOfResources; - case hipErrorInvalidValue: - return cudaErrorInvalidValue; - case hipErrorInvalidHandle: - return cudaErrorInvalidResourceHandle; - case hipErrorInvalidDevice: - return cudaErrorInvalidDevice; - case hipErrorInvalidMemcpyDirection: - return cudaErrorInvalidMemcpyDirection; - case hipErrorInvalidDevicePointer: - return cudaErrorInvalidDevicePointer; - case hipErrorNotInitialized: - return cudaErrorInitializationError; - case hipErrorNoDevice: - return cudaErrorNoDevice; - case hipErrorNotReady: - return cudaErrorNotReady; - case hipErrorPeerAccessNotEnabled: - return cudaErrorPeerAccessNotEnabled; - case hipErrorPeerAccessAlreadyEnabled: - return cudaErrorPeerAccessAlreadyEnabled; - case hipErrorHostMemoryAlreadyRegistered: - return cudaErrorHostMemoryAlreadyRegistered; - case hipErrorHostMemoryNotRegistered: - return cudaErrorHostMemoryNotRegistered; - case hipErrorDeinitialized: - return cudaErrorCudartUnloading; - case hipErrorInvalidSymbol: - return cudaErrorInvalidSymbol; - case hipErrorInsufficientDriver: - return cudaErrorInsufficientDriver; - case hipErrorMissingConfiguration: - return cudaErrorMissingConfiguration; - case hipErrorPriorLaunchFailure: - return cudaErrorPriorLaunchFailure; - case hipErrorInvalidDeviceFunction: - return cudaErrorInvalidDeviceFunction; - case hipErrorInvalidImage: - return cudaErrorInvalidKernelImage; - case hipErrorInvalidContext: -#if CUDA_VERSION >= 10020 - return cudaErrorDeviceUninitialized; -#else - return cudaErrorUnknown; -#endif - case hipErrorMapFailed: - return cudaErrorMapBufferObjectFailed; - case hipErrorUnmapFailed: - return cudaErrorUnmapBufferObjectFailed; - case hipErrorArrayIsMapped: -#if CUDA_VERSION >= 10010 - return cudaErrorArrayIsMapped; -#else - return cudaErrorUnknown; -#endif - case hipErrorAlreadyMapped: -#if CUDA_VERSION >= 10010 - return cudaErrorAlreadyMapped; -#else - return cudaErrorUnknown; -#endif - case hipErrorNoBinaryForGpu: - return cudaErrorNoKernelImageForDevice; - case hipErrorAlreadyAcquired: -#if CUDA_VERSION >= 10010 - return cudaErrorAlreadyAcquired; -#else - return cudaErrorUnknown; -#endif - case hipErrorNotMapped: -#if CUDA_VERSION >= 10010 - return cudaErrorNotMapped; -#else - return cudaErrorUnknown; -#endif - case hipErrorNotMappedAsArray: -#if CUDA_VERSION >= 10010 - return cudaErrorNotMappedAsArray; -#else - return cudaErrorUnknown; -#endif - case hipErrorNotMappedAsPointer: -#if CUDA_VERSION >= 10010 - return cudaErrorNotMappedAsPointer; -#else - return cudaErrorUnknown; -#endif - case hipErrorECCNotCorrectable: - return cudaErrorECCUncorrectable; - case hipErrorUnsupportedLimit: - return cudaErrorUnsupportedLimit; - case hipErrorContextAlreadyInUse: - return cudaErrorDeviceAlreadyInUse; - case hipErrorPeerAccessUnsupported: - return cudaErrorPeerAccessUnsupported; - case hipErrorInvalidKernelFile: - return cudaErrorInvalidPtx; - case hipErrorInvalidGraphicsContext: - return cudaErrorInvalidGraphicsContext; - case hipErrorInvalidSource: -#if CUDA_VERSION >= 10010 - return cudaErrorInvalidSource; -#else - return cudaErrorUnknown; -#endif - case hipErrorFileNotFound: -#if CUDA_VERSION >= 10010 - return cudaErrorFileNotFound; -#else - return cudaErrorUnknown; -#endif - case hipErrorSharedObjectSymbolNotFound: - return cudaErrorSharedObjectSymbolNotFound; - case hipErrorSharedObjectInitFailed: - return cudaErrorSharedObjectInitFailed; - case hipErrorOperatingSystem: - return cudaErrorOperatingSystem; - case hipErrorNotFound: -#if CUDA_VERSION >= 10010 - return cudaErrorSymbolNotFound; -#else - return cudaErrorUnknown; -#endif - case hipErrorIllegalAddress: - return cudaErrorIllegalAddress; - case hipErrorLaunchTimeOut: - return cudaErrorLaunchTimeout; - case hipErrorSetOnActiveProcess: - return cudaErrorSetOnActiveProcess; - case hipErrorLaunchFailure: - return cudaErrorLaunchFailure; - case hipErrorCooperativeLaunchTooLarge: - return cudaErrorCooperativeLaunchTooLarge; - case hipErrorNotSupported: - return cudaErrorNotSupported; - // HSA: does not exist in CUDA - case hipErrorRuntimeMemory: - // HSA: does not exist in CUDA - case hipErrorRuntimeOther: - case hipErrorUnknown: - case hipErrorTbd: - default: - return cudaErrorUnknown; // Note - translated error. - } -} - -inline static enum cudaMemcpyKind hipMemcpyKindToCudaMemcpyKind(hipMemcpyKind kind) { - switch (kind) { - case hipMemcpyHostToHost: - return cudaMemcpyHostToHost; - case hipMemcpyHostToDevice: - return cudaMemcpyHostToDevice; - case hipMemcpyDeviceToHost: - return cudaMemcpyDeviceToHost; - case hipMemcpyDeviceToDevice: - return cudaMemcpyDeviceToDevice; - default: - return cudaMemcpyDefault; - } -} - -inline static enum cudaTextureAddressMode hipTextureAddressModeToCudaTextureAddressMode( - hipTextureAddressMode kind) { - switch (kind) { - case hipAddressModeWrap: - return cudaAddressModeWrap; - case hipAddressModeClamp: - return cudaAddressModeClamp; - case hipAddressModeMirror: - return cudaAddressModeMirror; - case hipAddressModeBorder: - return cudaAddressModeBorder; - default: - return cudaAddressModeWrap; - } -} - -inline static enum cudaMemRangeAttribute hipMemRangeAttributeToCudaMemRangeAttribute( - hipMemRangeAttribute kind) { - switch (kind) { - case hipMemRangeAttributeReadMostly: - return cudaMemRangeAttributeReadMostly; - case hipMemRangeAttributePreferredLocation: - return cudaMemRangeAttributePreferredLocation; - case hipMemRangeAttributeAccessedBy: - return cudaMemRangeAttributeAccessedBy; - case hipMemRangeAttributeLastPrefetchLocation: - return cudaMemRangeAttributeLastPrefetchLocation; - default: - return cudaMemRangeAttributeReadMostly; - } -} - -inline static enum cudaMemoryAdvise hipMemoryAdviseTocudaMemoryAdvise( - hipMemoryAdvise kind) { - switch (kind) { - case hipMemAdviseSetReadMostly: - return cudaMemAdviseSetReadMostly; - case hipMemAdviseUnsetReadMostly : - return cudaMemAdviseUnsetReadMostly ; - case hipMemAdviseSetPreferredLocation: - return cudaMemAdviseSetPreferredLocation; - case hipMemAdviseUnsetPreferredLocation: - return cudaMemAdviseUnsetPreferredLocation; - case hipMemAdviseSetAccessedBy: - return cudaMemAdviseSetAccessedBy; - case hipMemAdviseUnsetAccessedBy: - return cudaMemAdviseUnsetAccessedBy; - default: - return cudaMemAdviseSetReadMostly; - } -} - -inline static enum cudaTextureFilterMode hipTextureFilterModeToCudaTextureFilterMode( - hipTextureFilterMode kind) { - switch (kind) { - case hipFilterModePoint: - return cudaFilterModePoint; - case hipFilterModeLinear: - return cudaFilterModeLinear; - default: - return cudaFilterModePoint; - } -} - -inline static enum cudaTextureReadMode hipTextureReadModeToCudaTextureReadMode(hipTextureReadMode kind) { - switch (kind) { - case hipReadModeElementType: - return cudaReadModeElementType; - case hipReadModeNormalizedFloat: - return cudaReadModeNormalizedFloat; - default: - return cudaReadModeElementType; - } -} - -inline static enum cudaChannelFormatKind hipChannelFormatKindToCudaChannelFormatKind( - hipChannelFormatKind kind) { - switch (kind) { - case hipChannelFormatKindSigned: - return cudaChannelFormatKindSigned; - case hipChannelFormatKindUnsigned: - return cudaChannelFormatKindUnsigned; - case hipChannelFormatKindFloat: - return cudaChannelFormatKindFloat; - case hipChannelFormatKindNone: - return cudaChannelFormatKindNone; - default: - return cudaChannelFormatKindNone; - } -} - -/** - * Stream CallBack struct - */ -#define HIPRT_CB CUDART_CB -typedef void(HIPRT_CB* hipStreamCallback_t)(hipStream_t stream, hipError_t status, void* userData); -inline static hipError_t hipInit(unsigned int flags) { - return hipCUResultTohipError(cuInit(flags)); -} - -inline static hipError_t hipDeviceReset() { return hipCUDAErrorTohipError(cudaDeviceReset()); } - -inline static hipError_t hipGetLastError() { return hipCUDAErrorTohipError(cudaGetLastError()); } - -inline static hipError_t hipPeekAtLastError() { - return hipCUDAErrorTohipError(cudaPeekAtLastError()); -} - -inline static hipError_t hipMalloc(void** ptr, size_t size) { - return hipCUDAErrorTohipError(cudaMalloc(ptr, size)); -} - -inline static hipError_t hipMallocPitch(void** ptr, size_t* pitch, size_t width, size_t height) { - return hipCUDAErrorTohipError(cudaMallocPitch(ptr, pitch, width, height)); -} - -inline static hipError_t hipMemAllocPitch(hipDeviceptr_t* dptr,size_t* pitch,size_t widthInBytes,size_t height,unsigned int elementSizeBytes){ - return hipCUResultTohipError(cuMemAllocPitch(dptr,pitch,widthInBytes,height,elementSizeBytes)); -} - -inline static hipError_t hipMalloc3D(hipPitchedPtr* pitchedDevPtr, hipExtent extent) { - return hipCUDAErrorTohipError(cudaMalloc3D(pitchedDevPtr, extent)); -} - -inline static hipError_t hipFree(void* ptr) { return hipCUDAErrorTohipError(cudaFree(ptr)); } - -inline static hipError_t hipMallocHost(void** ptr, size_t size) - __attribute__((deprecated("use hipHostMalloc instead"))); -inline static hipError_t hipMallocHost(void** ptr, size_t size) { - return hipCUDAErrorTohipError(cudaMallocHost(ptr, size)); -} - -inline static hipError_t hipMemAllocHost(void** ptr, size_t size) - __attribute__((deprecated("use hipHostMalloc instead"))); -inline static hipError_t hipMemAllocHost(void** ptr, size_t size) { - return hipCUResultTohipError(cuMemAllocHost(ptr, size)); -} - -inline static hipError_t hipHostAlloc(void** ptr, size_t size, unsigned int flags) - __attribute__((deprecated("use hipHostMalloc instead"))); -inline static hipError_t hipHostAlloc(void** ptr, size_t size, unsigned int flags) { - return hipCUDAErrorTohipError(cudaHostAlloc(ptr, size, flags)); -} - -inline static hipError_t hipHostMalloc(void** ptr, size_t size, unsigned int flags) { - return hipCUDAErrorTohipError(cudaHostAlloc(ptr, size, flags)); -} - -inline static hipError_t hipMemAdvise(const void* dev_ptr, size_t count, hipMemoryAdvise advice, - int device) { - return hipCUDAErrorTohipError(cudaMemAdvise(dev_ptr, count, - hipMemoryAdviseTocudaMemoryAdvise(advice), device)); -} - -inline static hipError_t hipMemPrefetchAsync(const void* dev_ptr, size_t count, int device, - hipStream_t stream __dparm(0)) { - return hipCUDAErrorTohipError(cudaMemPrefetchAsync(dev_ptr, count, device, stream)); -} - -inline static hipError_t hipMemRangeGetAttribute(void* data, size_t data_size, - hipMemRangeAttribute attribute, - const void* dev_ptr, size_t count) { - return hipCUDAErrorTohipError(cudaMemRangeGetAttribute(data, data_size, - hipMemRangeAttributeToCudaMemRangeAttribute(attribute), dev_ptr, count)); -} - -inline static hipError_t hipMemRangeGetAttributes(void** data, size_t* data_sizes, - hipMemRangeAttribute* attributes, - size_t num_attributes, const void* dev_ptr, - size_t count) { - return hipCUDAErrorTohipError(cudaMemRangeGetAttributes(data, data_sizes, attributes, - num_attributes, dev_ptr, count)); -} - -inline static hipError_t hipStreamAttachMemAsync(hipStream_t stream, hipDeviceptr_t* dev_ptr, - size_t length __dparm(0), - unsigned int flags __dparm(hipMemAttachSingle)) { - return hipCUDAErrorTohipError(cudaStreamAttachMemAsync(stream, dev_ptr, length, flags)); -} - -inline static hipError_t hipMallocManaged(void** ptr, size_t size, unsigned int flags) { - return hipCUDAErrorTohipError(cudaMallocManaged(ptr, size, flags)); -} - -inline static hipError_t hipMallocArray(hipArray** array, const hipChannelFormatDesc* desc, - size_t width, size_t height, - unsigned int flags __dparm(hipArrayDefault)) { - return hipCUDAErrorTohipError(cudaMallocArray(array, desc, width, height, flags)); -} - -inline static hipError_t hipMalloc3DArray(hipArray** array, const hipChannelFormatDesc* desc, - hipExtent extent, unsigned int flags) { - return hipCUDAErrorTohipError(cudaMalloc3DArray(array, desc, extent, flags)); -} - -inline static hipError_t hipFreeArray(hipArray* array) { - return hipCUDAErrorTohipError(cudaFreeArray(array)); -} - -inline static hipError_t hipHostGetDevicePointer(void** devPtr, void* hostPtr, unsigned int flags) { - return hipCUDAErrorTohipError(cudaHostGetDevicePointer(devPtr, hostPtr, flags)); -} - -inline static hipError_t hipHostGetFlags(unsigned int* flagsPtr, void* hostPtr) { - return hipCUDAErrorTohipError(cudaHostGetFlags(flagsPtr, hostPtr)); -} - -inline static hipError_t hipHostRegister(void* ptr, size_t size, unsigned int flags) { - return hipCUDAErrorTohipError(cudaHostRegister(ptr, size, flags)); -} - -inline static hipError_t hipHostUnregister(void* ptr) { - return hipCUDAErrorTohipError(cudaHostUnregister(ptr)); -} - -inline static hipError_t hipFreeHost(void* ptr) - __attribute__((deprecated("use hipHostFree instead"))); -inline static hipError_t hipFreeHost(void* ptr) { - return hipCUDAErrorTohipError(cudaFreeHost(ptr)); -} - -inline static hipError_t hipHostFree(void* ptr) { - return hipCUDAErrorTohipError(cudaFreeHost(ptr)); -} - -inline static hipError_t hipSetDevice(int device) { - return hipCUDAErrorTohipError(cudaSetDevice(device)); -} - -inline static hipError_t hipChooseDevice(int* device, const hipDeviceProp_t* prop) { - struct cudaDeviceProp cdprop; - memset(&cdprop, 0x0, sizeof(struct cudaDeviceProp)); - cdprop.major = prop->major; - cdprop.minor = prop->minor; - cdprop.totalGlobalMem = prop->totalGlobalMem; - cdprop.sharedMemPerBlock = prop->sharedMemPerBlock; - cdprop.regsPerBlock = prop->regsPerBlock; - cdprop.warpSize = prop->warpSize; - cdprop.maxThreadsPerBlock = prop->maxThreadsPerBlock; - cdprop.clockRate = prop->clockRate; - cdprop.totalConstMem = prop->totalConstMem; - cdprop.multiProcessorCount = prop->multiProcessorCount; - cdprop.l2CacheSize = prop->l2CacheSize; - cdprop.maxThreadsPerMultiProcessor = prop->maxThreadsPerMultiProcessor; - cdprop.computeMode = prop->computeMode; - cdprop.canMapHostMemory = prop->canMapHostMemory; - cdprop.memoryClockRate = prop->memoryClockRate; - cdprop.memoryBusWidth = prop->memoryBusWidth; - return hipCUDAErrorTohipError(cudaChooseDevice(device, &cdprop)); -} - -inline static hipError_t hipMemcpyHtoD(hipDeviceptr_t dst, void* src, size_t size) { - return hipCUResultTohipError(cuMemcpyHtoD(dst, src, size)); -} - -inline static hipError_t hipMemcpyDtoH(void* dst, hipDeviceptr_t src, size_t size) { - return hipCUResultTohipError(cuMemcpyDtoH(dst, src, size)); -} - -inline static hipError_t hipMemcpyDtoD(hipDeviceptr_t dst, hipDeviceptr_t src, size_t size) { - return hipCUResultTohipError(cuMemcpyDtoD(dst, src, size)); -} - -inline static hipError_t hipMemcpyHtoDAsync(hipDeviceptr_t dst, void* src, size_t size, - hipStream_t stream) { - return hipCUResultTohipError(cuMemcpyHtoDAsync(dst, src, size, stream)); -} - -inline static hipError_t hipMemcpyDtoHAsync(void* dst, hipDeviceptr_t src, size_t size, - hipStream_t stream) { - return hipCUResultTohipError(cuMemcpyDtoHAsync(dst, src, size, stream)); -} - -inline static hipError_t hipMemcpyDtoDAsync(hipDeviceptr_t dst, hipDeviceptr_t src, size_t size, - hipStream_t stream) { - return hipCUResultTohipError(cuMemcpyDtoDAsync(dst, src, size, stream)); -} - -inline static hipError_t hipMemcpy(void* dst, const void* src, size_t sizeBytes, - hipMemcpyKind copyKind) { - return hipCUDAErrorTohipError( - cudaMemcpy(dst, src, sizeBytes, hipMemcpyKindToCudaMemcpyKind(copyKind))); -} - - -inline static hipError_t hipMemcpyWithStream(void* dst, const void* src, - size_t sizeBytes, hipMemcpyKind copyKind, - hipStream_t stream) { - cudaError_t error = cudaMemcpyAsync(dst, src, sizeBytes, - hipMemcpyKindToCudaMemcpyKind(copyKind), - stream); - - if (error != cudaSuccess) return hipCUDAErrorTohipError(error); - - return hipCUDAErrorTohipError(cudaStreamSynchronize(stream)); -} - -inline static hipError_t hipMemcpyAsync(void* dst, const void* src, size_t sizeBytes, - hipMemcpyKind copyKind, hipStream_t stream __dparm(0)) { - return hipCUDAErrorTohipError( - cudaMemcpyAsync(dst, src, sizeBytes, hipMemcpyKindToCudaMemcpyKind(copyKind), stream)); -} - -inline static hipError_t hipMemcpyToSymbol(const void* symbol, const void* src, size_t sizeBytes, - size_t offset __dparm(0), - hipMemcpyKind copyType __dparm(hipMemcpyHostToDevice)) { - return hipCUDAErrorTohipError(cudaMemcpyToSymbol(symbol, src, sizeBytes, offset, - hipMemcpyKindToCudaMemcpyKind(copyType))); -} - -inline static hipError_t hipMemcpyToSymbolAsync(const void* symbol, const void* src, - size_t sizeBytes, size_t offset, - hipMemcpyKind copyType, - hipStream_t stream __dparm(0)) { - return hipCUDAErrorTohipError(cudaMemcpyToSymbolAsync( - symbol, src, sizeBytes, offset, hipMemcpyKindToCudaMemcpyKind(copyType), stream)); -} - -inline static hipError_t hipMemcpyFromSymbol(void* dst, const void* symbolName, size_t sizeBytes, - size_t offset __dparm(0), - hipMemcpyKind kind __dparm(hipMemcpyDeviceToHost)) { - return hipCUDAErrorTohipError(cudaMemcpyFromSymbol(dst, symbolName, sizeBytes, offset, - hipMemcpyKindToCudaMemcpyKind(kind))); -} - -inline static hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* symbolName, - size_t sizeBytes, size_t offset, - hipMemcpyKind kind, - hipStream_t stream __dparm(0)) { - return hipCUDAErrorTohipError(cudaMemcpyFromSymbolAsync( - dst, symbolName, sizeBytes, offset, hipMemcpyKindToCudaMemcpyKind(kind), stream)); -} - -inline static hipError_t hipGetSymbolAddress(void** devPtr, const void* symbolName) { - return hipCUDAErrorTohipError(cudaGetSymbolAddress(devPtr, symbolName)); -} - -inline static hipError_t hipGetSymbolSize(size_t* size, const void* symbolName) { - return hipCUDAErrorTohipError(cudaGetSymbolSize(size, symbolName)); -} - -inline static hipError_t hipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, - size_t width, size_t height, hipMemcpyKind kind) { - return hipCUDAErrorTohipError( - cudaMemcpy2D(dst, dpitch, src, spitch, width, height, hipMemcpyKindToCudaMemcpyKind(kind))); -} - -inline static hipError_t hipMemcpyParam2D(const hip_Memcpy2D* pCopy) { - return hipCUResultTohipError(cuMemcpy2D(pCopy)); -} - -inline static hipError_t hipMemcpyParam2DAsync(const hip_Memcpy2D* pCopy, hipStream_t stream __dparm(0)) { - return hipCUResultTohipError(cuMemcpy2DAsync(pCopy, stream)); -} - -inline static hipError_t hipMemcpy3D(const struct hipMemcpy3DParms *p) { - return hipCUDAErrorTohipError(cudaMemcpy3D(p)); -} - -inline static hipError_t hipMemcpy3DAsync(const struct hipMemcpy3DParms *p, hipStream_t stream) { - return hipCUDAErrorTohipError(cudaMemcpy3DAsync(p, stream)); -} - -inline static hipError_t hipDrvMemcpy3D(const HIP_MEMCPY3D* pCopy) { - return hipCUResultTohipError(cuMemcpy3D(pCopy)); -} - -inline static hipError_t hipDrvMemcpy3DAsync(const HIP_MEMCPY3D* pCopy, hipStream_t stream) { - return hipCUResultTohipError(cuMemcpy3DAsync(pCopy, stream)); -} - -inline static hipError_t hipMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, - size_t width, size_t height, hipMemcpyKind kind, - hipStream_t stream) { - return hipCUDAErrorTohipError(cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, - hipMemcpyKindToCudaMemcpyKind(kind), stream)); -} - -inline static hipError_t hipMemcpy2DFromArray(void* dst, size_t dpitch, hipArray* src, - size_t wOffset, size_t hOffset, size_t width, - size_t height, hipMemcpyKind kind) { - return hipCUDAErrorTohipError(cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, - height, - hipMemcpyKindToCudaMemcpyKind(kind))); -} - -inline static hipError_t hipMemcpy2DFromArrayAsync(void* dst, size_t dpitch, hipArray* src, - size_t wOffset, size_t hOffset, size_t width, - size_t height, hipMemcpyKind kind, - hipStream_t stream) { - return hipCUDAErrorTohipError(cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, - width, height, - hipMemcpyKindToCudaMemcpyKind(kind), - stream)); -} - -inline static hipError_t hipMemcpy2DToArray(hipArray* dst, size_t wOffset, size_t hOffset, - const void* src, size_t spitch, size_t width, - size_t height, hipMemcpyKind kind) { - return hipCUDAErrorTohipError(cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, - height, hipMemcpyKindToCudaMemcpyKind(kind))); -} - -inline static hipError_t hipMemcpy2DToArrayAsync(hipArray* dst, size_t wOffset, size_t hOffset, - const void* src, size_t spitch, size_t width, - size_t height, hipMemcpyKind kind, - hipStream_t stream) { - return hipCUDAErrorTohipError(cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, - width, height, - hipMemcpyKindToCudaMemcpyKind(kind), - stream)); -} - -__HIP_DEPRECATED inline static hipError_t hipMemcpyToArray(hipArray* dst, size_t wOffset, - size_t hOffset, const void* src, - size_t count, hipMemcpyKind kind) { - return hipCUDAErrorTohipError( - cudaMemcpyToArray(dst, wOffset, hOffset, src, count, hipMemcpyKindToCudaMemcpyKind(kind))); -} - -__HIP_DEPRECATED inline static hipError_t hipMemcpyFromArray(void* dst, hipArray_const_t srcArray, - size_t wOffset, size_t hOffset, - size_t count, hipMemcpyKind kind) { - return hipCUDAErrorTohipError(cudaMemcpyFromArray(dst, srcArray, wOffset, hOffset, count, - hipMemcpyKindToCudaMemcpyKind(kind))); -} - -inline static hipError_t hipMemcpyAtoH(void* dst, hipArray* srcArray, size_t srcOffset, - size_t count) { - return hipCUResultTohipError(cuMemcpyAtoH(dst, (CUarray)srcArray, srcOffset, count)); -} - -inline static hipError_t hipMemcpyHtoA(hipArray* dstArray, size_t dstOffset, const void* srcHost, - size_t count) { - return hipCUResultTohipError(cuMemcpyHtoA((CUarray)dstArray, dstOffset, srcHost, count)); -} - -inline static hipError_t hipDeviceSynchronize() { - return hipCUDAErrorTohipError(cudaDeviceSynchronize()); -} - -inline static hipError_t hipDeviceGetCacheConfig(hipFuncCache_t* pCacheConfig) { - return hipCUDAErrorTohipError(cudaDeviceGetCacheConfig(pCacheConfig)); -} - -inline static hipError_t hipFuncSetAttribute(const void* func, hipFuncAttribute attr, int value) { - return hipCUDAErrorTohipError(cudaFuncSetAttribute(func, attr, value)); -} - -inline static hipError_t hipDeviceSetCacheConfig(hipFuncCache_t cacheConfig) { - return hipCUDAErrorTohipError(cudaDeviceSetCacheConfig(cacheConfig)); -} - -inline static hipError_t hipFuncSetSharedMemConfig(const void* func, hipSharedMemConfig config) { - return hipCUDAErrorTohipError(cudaFuncSetSharedMemConfig(func, config)); -} - -inline static const char* hipGetErrorString(hipError_t error) { - return cudaGetErrorString(hipErrorToCudaError(error)); -} - -inline static const char* hipGetErrorName(hipError_t error) { - return cudaGetErrorName(hipErrorToCudaError(error)); -} - -inline static hipError_t hipGetDeviceCount(int* count) { - return hipCUDAErrorTohipError(cudaGetDeviceCount(count)); -} - -inline static hipError_t hipGetDevice(int* device) { - return hipCUDAErrorTohipError(cudaGetDevice(device)); -} - -inline static hipError_t hipIpcCloseMemHandle(void* devPtr) { - return hipCUDAErrorTohipError(cudaIpcCloseMemHandle(devPtr)); -} - -inline static hipError_t hipIpcGetEventHandle(hipIpcEventHandle_t* handle, hipEvent_t event) { - return hipCUDAErrorTohipError(cudaIpcGetEventHandle(handle, event)); -} - -inline static hipError_t hipIpcGetMemHandle(hipIpcMemHandle_t* handle, void* devPtr) { - return hipCUDAErrorTohipError(cudaIpcGetMemHandle(handle, devPtr)); -} - -inline static hipError_t hipIpcOpenEventHandle(hipEvent_t* event, hipIpcEventHandle_t handle) { - return hipCUDAErrorTohipError(cudaIpcOpenEventHandle(event, handle)); -} - -inline static hipError_t hipIpcOpenMemHandle(void** devPtr, hipIpcMemHandle_t handle, - unsigned int flags) { - return hipCUDAErrorTohipError(cudaIpcOpenMemHandle(devPtr, handle, flags)); -} - -inline static hipError_t hipMemset(void* devPtr, int value, size_t count) { - return hipCUDAErrorTohipError(cudaMemset(devPtr, value, count)); -} - -inline static hipError_t hipMemsetD32(hipDeviceptr_t devPtr, int value, size_t count) { - return hipCUResultTohipError(cuMemsetD32(devPtr, value, count)); -} - -inline static hipError_t hipMemsetAsync(void* devPtr, int value, size_t count, - hipStream_t stream __dparm(0)) { - return hipCUDAErrorTohipError(cudaMemsetAsync(devPtr, value, count, stream)); -} - -inline static hipError_t hipMemsetD32Async(hipDeviceptr_t devPtr, int value, size_t count, - hipStream_t stream __dparm(0)) { - return hipCUResultTohipError(cuMemsetD32Async(devPtr, value, count, stream)); -} - -inline static hipError_t hipMemsetD8(hipDeviceptr_t dest, unsigned char value, size_t sizeBytes) { - return hipCUResultTohipError(cuMemsetD8(dest, value, sizeBytes)); -} - -inline static hipError_t hipMemsetD8Async(hipDeviceptr_t dest, unsigned char value, size_t sizeBytes, - hipStream_t stream __dparm(0)) { - return hipCUResultTohipError(cuMemsetD8Async(dest, value, sizeBytes, stream)); -} - -inline static hipError_t hipMemsetD16(hipDeviceptr_t dest, unsigned short value, size_t sizeBytes) { - return hipCUResultTohipError(cuMemsetD16(dest, value, sizeBytes)); -} - -inline static hipError_t hipMemsetD16Async(hipDeviceptr_t dest, unsigned short value, size_t sizeBytes, - hipStream_t stream __dparm(0)) { - return hipCUResultTohipError(cuMemsetD16Async(dest, value, sizeBytes, stream)); -} - -inline static hipError_t hipMemset2D(void* dst, size_t pitch, int value, size_t width, size_t height) { - return hipCUDAErrorTohipError(cudaMemset2D(dst, pitch, value, width, height)); -} - -inline static hipError_t hipMemset2DAsync(void* dst, size_t pitch, int value, size_t width, size_t height, hipStream_t stream __dparm(0)) { - return hipCUDAErrorTohipError(cudaMemset2DAsync(dst, pitch, value, width, height, stream)); -} - -inline static hipError_t hipMemset3D(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent ){ - return hipCUDAErrorTohipError(cudaMemset3D(pitchedDevPtr, value, extent)); -} - -inline static hipError_t hipMemset3DAsync(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent, hipStream_t stream __dparm(0) ){ - return hipCUDAErrorTohipError(cudaMemset3DAsync(pitchedDevPtr, value, extent, stream)); -} - -inline static hipError_t hipGetDeviceProperties(hipDeviceProp_t* p_prop, int device) { - struct cudaDeviceProp cdprop; - cudaError_t cerror; - cerror = cudaGetDeviceProperties(&cdprop, device); - - strncpy(p_prop->name, cdprop.name, 256); - p_prop->totalGlobalMem = cdprop.totalGlobalMem; - p_prop->sharedMemPerBlock = cdprop.sharedMemPerBlock; - p_prop->regsPerBlock = cdprop.regsPerBlock; - p_prop->warpSize = cdprop.warpSize; - p_prop->maxThreadsPerBlock = cdprop.maxThreadsPerBlock; - for (int i = 0; i < 3; i++) { - p_prop->maxThreadsDim[i] = cdprop.maxThreadsDim[i]; - p_prop->maxGridSize[i] = cdprop.maxGridSize[i]; - } - p_prop->clockRate = cdprop.clockRate; - p_prop->memoryClockRate = cdprop.memoryClockRate; - p_prop->memoryBusWidth = cdprop.memoryBusWidth; - p_prop->totalConstMem = cdprop.totalConstMem; - p_prop->major = cdprop.major; - p_prop->minor = cdprop.minor; - p_prop->multiProcessorCount = cdprop.multiProcessorCount; - p_prop->l2CacheSize = cdprop.l2CacheSize; - p_prop->maxThreadsPerMultiProcessor = cdprop.maxThreadsPerMultiProcessor; - p_prop->computeMode = cdprop.computeMode; - p_prop->clockInstructionRate = cdprop.clockRate; // Same as clock-rate: - - int ccVers = p_prop->major * 100 + p_prop->minor * 10; - p_prop->arch.hasGlobalInt32Atomics = (ccVers >= 110); - p_prop->arch.hasGlobalFloatAtomicExch = (ccVers >= 110); - p_prop->arch.hasSharedInt32Atomics = (ccVers >= 120); - p_prop->arch.hasSharedFloatAtomicExch = (ccVers >= 120); - p_prop->arch.hasFloatAtomicAdd = (ccVers >= 200); - p_prop->arch.hasGlobalInt64Atomics = (ccVers >= 120); - p_prop->arch.hasSharedInt64Atomics = (ccVers >= 110); - p_prop->arch.hasDoubles = (ccVers >= 130); - p_prop->arch.hasWarpVote = (ccVers >= 120); - p_prop->arch.hasWarpBallot = (ccVers >= 200); - p_prop->arch.hasWarpShuffle = (ccVers >= 300); - p_prop->arch.hasFunnelShift = (ccVers >= 350); - p_prop->arch.hasThreadFenceSystem = (ccVers >= 200); - p_prop->arch.hasSyncThreadsExt = (ccVers >= 200); - p_prop->arch.hasSurfaceFuncs = (ccVers >= 200); - p_prop->arch.has3dGrid = (ccVers >= 200); - p_prop->arch.hasDynamicParallelism = (ccVers >= 350); - - p_prop->concurrentKernels = cdprop.concurrentKernels; - p_prop->pciDomainID = cdprop.pciDomainID; - p_prop->pciBusID = cdprop.pciBusID; - p_prop->pciDeviceID = cdprop.pciDeviceID; - p_prop->maxSharedMemoryPerMultiProcessor = cdprop.sharedMemPerMultiprocessor; - p_prop->isMultiGpuBoard = cdprop.isMultiGpuBoard; - p_prop->canMapHostMemory = cdprop.canMapHostMemory; - p_prop->gcnArch = 0; // Not a GCN arch - p_prop->integrated = cdprop.integrated; - p_prop->cooperativeLaunch = cdprop.cooperativeLaunch; - p_prop->cooperativeMultiDeviceLaunch = cdprop.cooperativeMultiDeviceLaunch; - p_prop->cooperativeMultiDeviceUnmatchedFunc = 0; - p_prop->cooperativeMultiDeviceUnmatchedGridDim = 0; - p_prop->cooperativeMultiDeviceUnmatchedBlockDim = 0; - p_prop->cooperativeMultiDeviceUnmatchedSharedMem = 0; - - p_prop->maxTexture1D = cdprop.maxTexture1D; - p_prop->maxTexture2D[0] = cdprop.maxTexture2D[0]; - p_prop->maxTexture2D[1] = cdprop.maxTexture2D[1]; - p_prop->maxTexture3D[0] = cdprop.maxTexture3D[0]; - p_prop->maxTexture3D[1] = cdprop.maxTexture3D[1]; - p_prop->maxTexture3D[2] = cdprop.maxTexture3D[2]; - - p_prop->memPitch = cdprop.memPitch; - p_prop->textureAlignment = cdprop.textureAlignment; - p_prop->texturePitchAlignment = cdprop.texturePitchAlignment; - p_prop->kernelExecTimeoutEnabled = cdprop.kernelExecTimeoutEnabled; - p_prop->ECCEnabled = cdprop.ECCEnabled; - p_prop->tccDriver = cdprop.tccDriver; - - return hipCUDAErrorTohipError(cerror); -} - -inline static hipError_t hipDeviceGetAttribute(int* pi, hipDeviceAttribute_t attr, int device) { - enum cudaDeviceAttr cdattr; - cudaError_t cerror; - - switch (attr) { - case hipDeviceAttributeMaxThreadsPerBlock: - cdattr = cudaDevAttrMaxThreadsPerBlock; - break; - case hipDeviceAttributeMaxBlockDimX: - cdattr = cudaDevAttrMaxBlockDimX; - break; - case hipDeviceAttributeMaxBlockDimY: - cdattr = cudaDevAttrMaxBlockDimY; - break; - case hipDeviceAttributeMaxBlockDimZ: - cdattr = cudaDevAttrMaxBlockDimZ; - break; - case hipDeviceAttributeMaxGridDimX: - cdattr = cudaDevAttrMaxGridDimX; - break; - case hipDeviceAttributeMaxGridDimY: - cdattr = cudaDevAttrMaxGridDimY; - break; - case hipDeviceAttributeMaxGridDimZ: - cdattr = cudaDevAttrMaxGridDimZ; - break; - case hipDeviceAttributeMaxSharedMemoryPerBlock: - cdattr = cudaDevAttrMaxSharedMemoryPerBlock; - break; - case hipDeviceAttributeTotalConstantMemory: - cdattr = cudaDevAttrTotalConstantMemory; - break; - case hipDeviceAttributeWarpSize: - cdattr = cudaDevAttrWarpSize; - break; - case hipDeviceAttributeMaxRegistersPerBlock: - cdattr = cudaDevAttrMaxRegistersPerBlock; - break; - case hipDeviceAttributeClockRate: - cdattr = cudaDevAttrClockRate; - break; - case hipDeviceAttributeMemoryClockRate: - cdattr = cudaDevAttrMemoryClockRate; - break; - case hipDeviceAttributeMemoryBusWidth: - cdattr = cudaDevAttrGlobalMemoryBusWidth; - break; - case hipDeviceAttributeMultiprocessorCount: - cdattr = cudaDevAttrMultiProcessorCount; - break; - case hipDeviceAttributeComputeMode: - cdattr = cudaDevAttrComputeMode; - break; - case hipDeviceAttributeL2CacheSize: - cdattr = cudaDevAttrL2CacheSize; - break; - case hipDeviceAttributeMaxThreadsPerMultiProcessor: - cdattr = cudaDevAttrMaxThreadsPerMultiProcessor; - break; - case hipDeviceAttributeComputeCapabilityMajor: - cdattr = cudaDevAttrComputeCapabilityMajor; - break; - case hipDeviceAttributeComputeCapabilityMinor: - cdattr = cudaDevAttrComputeCapabilityMinor; - break; - case hipDeviceAttributeConcurrentKernels: - cdattr = cudaDevAttrConcurrentKernels; - break; - case hipDeviceAttributePciBusId: - cdattr = cudaDevAttrPciBusId; - break; - case hipDeviceAttributePciDeviceId: - cdattr = cudaDevAttrPciDeviceId; - break; - case hipDeviceAttributeMaxSharedMemoryPerMultiprocessor: - cdattr = cudaDevAttrMaxSharedMemoryPerMultiprocessor; - break; - case hipDeviceAttributeIsMultiGpuBoard: - cdattr = cudaDevAttrIsMultiGpuBoard; - break; - case hipDeviceAttributeIntegrated: - cdattr = cudaDevAttrIntegrated; - break; - case hipDeviceAttributeMaxTexture1DWidth: - cdattr = cudaDevAttrMaxTexture1DWidth; - break; - case hipDeviceAttributeMaxTexture2DWidth: - cdattr = cudaDevAttrMaxTexture2DWidth; - break; - case hipDeviceAttributeMaxTexture2DHeight: - cdattr = cudaDevAttrMaxTexture2DHeight; - break; - case hipDeviceAttributeMaxTexture3DWidth: - cdattr = cudaDevAttrMaxTexture3DWidth; - break; - case hipDeviceAttributeMaxTexture3DHeight: - cdattr = cudaDevAttrMaxTexture3DHeight; - break; - case hipDeviceAttributeMaxTexture3DDepth: - cdattr = cudaDevAttrMaxTexture3DDepth; - break; - case hipDeviceAttributeMaxPitch: - cdattr = cudaDevAttrMaxPitch; - break; - case hipDeviceAttributeTextureAlignment: - cdattr = cudaDevAttrTextureAlignment; - break; - case hipDeviceAttributeTexturePitchAlignment: - cdattr = cudaDevAttrTexturePitchAlignment; - break; - case hipDeviceAttributeKernelExecTimeout: - cdattr = cudaDevAttrKernelExecTimeout; - break; - case hipDeviceAttributeCanMapHostMemory: - cdattr = cudaDevAttrCanMapHostMemory; - break; - case hipDeviceAttributeEccEnabled: - cdattr = cudaDevAttrEccEnabled; - break; - case hipDeviceAttributeCooperativeLaunch: - cdattr = cudaDevAttrCooperativeLaunch; - break; - case hipDeviceAttributeCooperativeMultiDeviceLaunch: - cdattr = cudaDevAttrCooperativeMultiDeviceLaunch; - break; - case hipDeviceAttributeConcurrentManagedAccess: - cdattr = cudaDevAttrConcurrentManagedAccess; - break; - case hipDeviceAttributeManagedMemory: - cdattr = cudaDevAttrManagedMemory; - break; - case hipDeviceAttributePageableMemoryAccessUsesHostPageTables: - cdattr = cudaDevAttrPageableMemoryAccessUsesHostPageTables; - break; - case hipDeviceAttributePageableMemoryAccess: - cdattr = cudaDevAttrPageableMemoryAccess; - break; - case hipDeviceAttributeDirectManagedMemAccessFromHost: - cdattr = cudaDevAttrDirectManagedMemAccessFromHost; - break; - default: - return hipCUDAErrorTohipError(cudaErrorInvalidValue); - } - - cerror = cudaDeviceGetAttribute(pi, cdattr, device); - - return hipCUDAErrorTohipError(cerror); -} - -inline static hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, - const void* func, - int blockSize, - size_t dynamicSMemSize) { - return hipCUDAErrorTohipError(cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, - blockSize, dynamicSMemSize)); -} - -inline static hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, - const void* func, - int blockSize, - size_t dynamicSMemSize, - unsigned int flags) { - return hipCUDAErrorTohipError(cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, - blockSize, dynamicSMemSize, flags)); -} - -inline static hipError_t hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, - hipFunction_t f, - int blockSize, - size_t dynamicSMemSize ){ - return hipCUResultTohipError(cuOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, f, - blockSize, dynamicSMemSize)); -} - -inline static hipError_t hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, - hipFunction_t f, - int blockSize, - size_t dynamicSMemSize, - unsigned int flags ) { - return hipCUResultTohipError(cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks,f, - blockSize, dynamicSMemSize, flags)); -} - -//TODO - Match CUoccupancyB2DSize -inline static hipError_t hipModuleOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize, - hipFunction_t f, size_t dynSharedMemPerBlk, - int blockSizeLimit){ - return hipCUResultTohipError(cuOccupancyMaxPotentialBlockSize(gridSize, blockSize, f, NULL, - dynSharedMemPerBlk, blockSizeLimit)); -} - -//TODO - Match CUoccupancyB2DSize -inline static hipError_t hipModuleOccupancyMaxPotentialBlockSizeWithFlags(int* gridSize, int* blockSize, - hipFunction_t f, size_t dynSharedMemPerBlk, - int blockSizeLimit, unsigned int flags){ - return hipCUResultTohipError(cuOccupancyMaxPotentialBlockSizeWithFlags(gridSize, blockSize, f, NULL, - dynSharedMemPerBlk, blockSizeLimit, flags)); -} - -inline static hipError_t hipPointerGetAttributes(hipPointerAttribute_t* attributes, const void* ptr) { - struct cudaPointerAttributes cPA; - hipError_t err = hipCUDAErrorTohipError(cudaPointerGetAttributes(&cPA, ptr)); - if (err == hipSuccess) { -#if (CUDART_VERSION >= 11000) - auto memType = cPA.type; -#else - unsigned memType = cPA.memoryType; // No auto because cuda 10.2 doesnt force c++11 -#endif - switch (memType) { - case cudaMemoryTypeDevice: - attributes->memoryType = hipMemoryTypeDevice; - break; - case cudaMemoryTypeHost: - attributes->memoryType = hipMemoryTypeHost; - break; - default: - return hipErrorUnknown; - } - attributes->device = cPA.device; - attributes->devicePointer = cPA.devicePointer; - attributes->hostPointer = cPA.hostPointer; - attributes->isManaged = 0; - attributes->allocationFlags = 0; - } - return err; -} - -inline static hipError_t hipMemGetInfo(size_t* free, size_t* total) { - return hipCUDAErrorTohipError(cudaMemGetInfo(free, total)); -} - -inline static hipError_t hipEventCreate(hipEvent_t* event) { - return hipCUDAErrorTohipError(cudaEventCreate(event)); -} - -inline static hipError_t hipEventRecord(hipEvent_t event, hipStream_t stream __dparm(NULL)) { - return hipCUDAErrorTohipError(cudaEventRecord(event, stream)); -} - -inline static hipError_t hipEventSynchronize(hipEvent_t event) { - return hipCUDAErrorTohipError(cudaEventSynchronize(event)); -} - -inline static hipError_t hipEventElapsedTime(float* ms, hipEvent_t start, hipEvent_t stop) { - return hipCUDAErrorTohipError(cudaEventElapsedTime(ms, start, stop)); -} - -inline static hipError_t hipEventDestroy(hipEvent_t event) { - return hipCUDAErrorTohipError(cudaEventDestroy(event)); -} - -inline static hipError_t hipStreamCreateWithFlags(hipStream_t* stream, unsigned int flags) { - return hipCUDAErrorTohipError(cudaStreamCreateWithFlags(stream, flags)); -} - -inline static hipError_t hipStreamCreateWithPriority(hipStream_t* stream, unsigned int flags, int priority) { - return hipCUDAErrorTohipError(cudaStreamCreateWithPriority(stream, flags, priority)); -} - -inline static hipError_t hipDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) { - return hipCUDAErrorTohipError(cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority)); -} - -inline static hipError_t hipStreamCreate(hipStream_t* stream) { - return hipCUDAErrorTohipError(cudaStreamCreate(stream)); -} - -inline static hipError_t hipStreamSynchronize(hipStream_t stream) { - return hipCUDAErrorTohipError(cudaStreamSynchronize(stream)); -} - -inline static hipError_t hipStreamDestroy(hipStream_t stream) { - return hipCUDAErrorTohipError(cudaStreamDestroy(stream)); -} - -inline static hipError_t hipStreamGetFlags(hipStream_t stream, unsigned int *flags) { - return hipCUDAErrorTohipError(cudaStreamGetFlags(stream, flags)); -} - -inline static hipError_t hipStreamGetPriority(hipStream_t stream, int *priority) { - return hipCUDAErrorTohipError(cudaStreamGetPriority(stream, priority)); -} - -inline static hipError_t hipStreamWaitEvent(hipStream_t stream, hipEvent_t event, - unsigned int flags) { - return hipCUDAErrorTohipError(cudaStreamWaitEvent(stream, event, flags)); -} - -inline static hipError_t hipStreamQuery(hipStream_t stream) { - return hipCUDAErrorTohipError(cudaStreamQuery(stream)); -} - -inline static hipError_t hipStreamAddCallback(hipStream_t stream, hipStreamCallback_t callback, - void* userData, unsigned int flags) { - return hipCUDAErrorTohipError( - cudaStreamAddCallback(stream, (cudaStreamCallback_t)callback, userData, flags)); -} - -inline static hipError_t hipDriverGetVersion(int* driverVersion) { - cudaError_t err = cudaDriverGetVersion(driverVersion); - - // Override driver version to match version reported on HCC side. - *driverVersion = 4; - - return hipCUDAErrorTohipError(err); -} - -inline static hipError_t hipRuntimeGetVersion(int* runtimeVersion) { - return hipCUDAErrorTohipError(cudaRuntimeGetVersion(runtimeVersion)); -} - -inline static hipError_t hipDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) { - return hipCUDAErrorTohipError(cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice)); -} - -inline static hipError_t hipDeviceDisablePeerAccess(int peerDevice) { - return hipCUDAErrorTohipError(cudaDeviceDisablePeerAccess(peerDevice)); -} - -inline static hipError_t hipDeviceEnablePeerAccess(int peerDevice, unsigned int flags) { - return hipCUDAErrorTohipError(cudaDeviceEnablePeerAccess(peerDevice, flags)); -} - -inline static hipError_t hipCtxDisablePeerAccess(hipCtx_t peerCtx) { - return hipCUResultTohipError(cuCtxDisablePeerAccess(peerCtx)); -} - -inline static hipError_t hipCtxEnablePeerAccess(hipCtx_t peerCtx, unsigned int flags) { - return hipCUResultTohipError(cuCtxEnablePeerAccess(peerCtx, flags)); -} - -inline static hipError_t hipDevicePrimaryCtxGetState(hipDevice_t dev, unsigned int* flags, - int* active) { - return hipCUResultTohipError(cuDevicePrimaryCtxGetState(dev, flags, active)); -} - -inline static hipError_t hipDevicePrimaryCtxRelease(hipDevice_t dev) { - return hipCUResultTohipError(cuDevicePrimaryCtxRelease(dev)); -} - -inline static hipError_t hipDevicePrimaryCtxRetain(hipCtx_t* pctx, hipDevice_t dev) { - return hipCUResultTohipError(cuDevicePrimaryCtxRetain(pctx, dev)); -} - -inline static hipError_t hipDevicePrimaryCtxReset(hipDevice_t dev) { - return hipCUResultTohipError(cuDevicePrimaryCtxReset(dev)); -} - -inline static hipError_t hipDevicePrimaryCtxSetFlags(hipDevice_t dev, unsigned int flags) { - return hipCUResultTohipError(cuDevicePrimaryCtxSetFlags(dev, flags)); -} - -inline static hipError_t hipMemGetAddressRange(hipDeviceptr_t* pbase, size_t* psize, - hipDeviceptr_t dptr) { - return hipCUResultTohipError(cuMemGetAddressRange(pbase, psize, dptr)); -} - -inline static hipError_t hipMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, - size_t count) { - return hipCUDAErrorTohipError(cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count)); -} - -inline static hipError_t hipMemcpyPeerAsync(void* dst, int dstDevice, const void* src, - int srcDevice, size_t count, - hipStream_t stream __dparm(0)) { - return hipCUDAErrorTohipError( - cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream)); -} - -// Profile APIs: -inline static hipError_t hipProfilerStart() { return hipCUDAErrorTohipError(cudaProfilerStart()); } - -inline static hipError_t hipProfilerStop() { return hipCUDAErrorTohipError(cudaProfilerStop()); } - -inline static hipError_t hipGetDeviceFlags(unsigned int* flags) { - return hipCUDAErrorTohipError(cudaGetDeviceFlags(flags)); -} - -inline static hipError_t hipSetDeviceFlags(unsigned int flags) { - return hipCUDAErrorTohipError(cudaSetDeviceFlags(flags)); -} - -inline static hipError_t hipEventCreateWithFlags(hipEvent_t* event, unsigned int flags) { - return hipCUDAErrorTohipError(cudaEventCreateWithFlags(event, flags)); -} - -inline static hipError_t hipEventQuery(hipEvent_t event) { - return hipCUDAErrorTohipError(cudaEventQuery(event)); -} - -inline static hipError_t hipCtxCreate(hipCtx_t* ctx, unsigned int flags, hipDevice_t device) { - return hipCUResultTohipError(cuCtxCreate(ctx, flags, device)); -} - -inline static hipError_t hipCtxDestroy(hipCtx_t ctx) { - return hipCUResultTohipError(cuCtxDestroy(ctx)); -} - -inline static hipError_t hipCtxPopCurrent(hipCtx_t* ctx) { - return hipCUResultTohipError(cuCtxPopCurrent(ctx)); -} - -inline static hipError_t hipCtxPushCurrent(hipCtx_t ctx) { - return hipCUResultTohipError(cuCtxPushCurrent(ctx)); -} - -inline static hipError_t hipCtxSetCurrent(hipCtx_t ctx) { - return hipCUResultTohipError(cuCtxSetCurrent(ctx)); -} - -inline static hipError_t hipCtxGetCurrent(hipCtx_t* ctx) { - return hipCUResultTohipError(cuCtxGetCurrent(ctx)); -} - -inline static hipError_t hipCtxGetDevice(hipDevice_t* device) { - return hipCUResultTohipError(cuCtxGetDevice(device)); -} - -inline static hipError_t hipCtxGetApiVersion(hipCtx_t ctx, int* apiVersion) { - return hipCUResultTohipError(cuCtxGetApiVersion(ctx, (unsigned int*)apiVersion)); -} - -inline static hipError_t hipCtxGetCacheConfig(hipFuncCache* cacheConfig) { - return hipCUResultTohipError(cuCtxGetCacheConfig(cacheConfig)); -} - -inline static hipError_t hipCtxSetCacheConfig(hipFuncCache cacheConfig) { - return hipCUResultTohipError(cuCtxSetCacheConfig(cacheConfig)); -} - -inline static hipError_t hipCtxSetSharedMemConfig(hipSharedMemConfig config) { - return hipCUResultTohipError(cuCtxSetSharedMemConfig((CUsharedconfig)config)); -} - -inline static hipError_t hipCtxGetSharedMemConfig(hipSharedMemConfig* pConfig) { - return hipCUResultTohipError(cuCtxGetSharedMemConfig((CUsharedconfig*)pConfig)); -} - -inline static hipError_t hipCtxSynchronize(void) { - return hipCUResultTohipError(cuCtxSynchronize()); -} - -inline static hipError_t hipCtxGetFlags(unsigned int* flags) { - return hipCUResultTohipError(cuCtxGetFlags(flags)); -} - -inline static hipError_t hipCtxDetach(hipCtx_t ctx) { - return hipCUResultTohipError(cuCtxDetach(ctx)); -} - -inline static hipError_t hipDeviceGet(hipDevice_t* device, int ordinal) { - return hipCUResultTohipError(cuDeviceGet(device, ordinal)); -} - -inline static hipError_t hipDeviceComputeCapability(int* major, int* minor, hipDevice_t device) { - return hipCUResultTohipError(cuDeviceComputeCapability(major, minor, device)); -} - -inline static hipError_t hipDeviceGetName(char* name, int len, hipDevice_t device) { - return hipCUResultTohipError(cuDeviceGetName(name, len, device)); -} - -inline static hipError_t hipDeviceGetP2PAttribute(int* value, hipDeviceP2PAttr attr, - int srcDevice, int dstDevice) { - return hipCUDAErrorTohipError(cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice)); -} - -inline static hipError_t hipDeviceGetPCIBusId(char* pciBusId, int len, hipDevice_t device) { - return hipCUDAErrorTohipError(cudaDeviceGetPCIBusId(pciBusId, len, device)); -} - -inline static hipError_t hipDeviceGetByPCIBusId(int* device, const char* pciBusId) { - return hipCUDAErrorTohipError(cudaDeviceGetByPCIBusId(device, pciBusId)); -} - -inline static hipError_t hipDeviceGetSharedMemConfig(hipSharedMemConfig* config) { - return hipCUDAErrorTohipError(cudaDeviceGetSharedMemConfig(config)); -} - -inline static hipError_t hipDeviceSetSharedMemConfig(hipSharedMemConfig config) { - return hipCUDAErrorTohipError(cudaDeviceSetSharedMemConfig(config)); -} - -inline static hipError_t hipDeviceGetLimit(size_t* pValue, hipLimit_t limit) { - return hipCUDAErrorTohipError(cudaDeviceGetLimit(pValue, limit)); -} - -inline static hipError_t hipDeviceTotalMem(size_t* bytes, hipDevice_t device) { - return hipCUResultTohipError(cuDeviceTotalMem(bytes, device)); -} - -inline static hipError_t hipModuleLoad(hipModule_t* module, const char* fname) { - return hipCUResultTohipError(cuModuleLoad(module, fname)); -} - -inline static hipError_t hipModuleUnload(hipModule_t hmod) { - return hipCUResultTohipError(cuModuleUnload(hmod)); -} - -inline static hipError_t hipModuleGetFunction(hipFunction_t* function, hipModule_t module, - const char* kname) { - return hipCUResultTohipError(cuModuleGetFunction(function, module, kname)); -} - -inline static hipError_t hipModuleGetTexRef(hipTexRef* pTexRef, hipModule_t hmod, const char* name){ - hipCUResultTohipError(cuModuleGetTexRef(pTexRef, hmod, name)); -} - -inline static hipError_t hipFuncGetAttributes(hipFuncAttributes* attr, const void* func) { - return hipCUDAErrorTohipError(cudaFuncGetAttributes(attr, func)); -} - -inline static hipError_t hipFuncGetAttribute (int* value, hipFunction_attribute attrib, hipFunction_t hfunc) { - return hipCUResultTohipError(cuFuncGetAttribute(value, attrib, hfunc)); -} - -inline static hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes, hipModule_t hmod, - const char* name) { - return hipCUResultTohipError(cuModuleGetGlobal(dptr, bytes, hmod, name)); -} - -inline static hipError_t hipModuleLoadData(hipModule_t* module, const void* image) { - return hipCUResultTohipError(cuModuleLoadData(module, image)); -} - -inline static hipError_t hipModuleLoadDataEx(hipModule_t* module, const void* image, - unsigned int numOptions, hipJitOption* options, - void** optionValues) { - return hipCUResultTohipError( - cuModuleLoadDataEx(module, image, numOptions, options, optionValues)); -} - -inline static hipError_t hipLaunchKernel(const void* function_address, dim3 numBlocks, - dim3 dimBlocks, void** args, size_t sharedMemBytes, - hipStream_t stream) -{ - return hipCUDAErrorTohipError(cudaLaunchKernel(function_address,numBlocks,dimBlocks,args,sharedMemBytes,stream)); -} - -inline static hipError_t hipModuleLaunchKernel(hipFunction_t f, unsigned int gridDimX, - unsigned int gridDimY, unsigned int gridDimZ, - unsigned int blockDimX, unsigned int blockDimY, - unsigned int blockDimZ, unsigned int sharedMemBytes, - hipStream_t stream, void** kernelParams, - void** extra) { - return hipCUResultTohipError(cuLaunchKernel(f, gridDimX, gridDimY, gridDimZ, blockDimX, - blockDimY, blockDimZ, sharedMemBytes, stream, - kernelParams, extra)); -} - -inline static hipError_t hipFuncSetCacheConfig(const void* func, hipFuncCache_t cacheConfig) { - return hipCUDAErrorTohipError(cudaFuncSetCacheConfig(func, cacheConfig)); -} - -__HIP_DEPRECATED inline static hipError_t hipBindTexture(size_t* offset, - struct textureReference* tex, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t size __dparm(UINT_MAX)) { - return hipCUDAErrorTohipError(cudaBindTexture(offset, tex, devPtr, desc, size)); -} - -__HIP_DEPRECATED inline static hipError_t hipBindTexture2D( - size_t* offset, struct textureReference* tex, const void* devPtr, - const hipChannelFormatDesc* desc, size_t width, size_t height, size_t pitch) { - return hipCUDAErrorTohipError(cudaBindTexture2D(offset, tex, devPtr, desc, width, height, pitch)); -} - -inline static hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, - hipChannelFormatKind f) { - return cudaCreateChannelDesc(x, y, z, w, hipChannelFormatKindToCudaChannelFormatKind(f)); -} - -inline static hipError_t hipCreateTextureObject(hipTextureObject_t* pTexObject, - const hipResourceDesc* pResDesc, - const hipTextureDesc* pTexDesc, - const hipResourceViewDesc* pResViewDesc) { - return hipCUDAErrorTohipError( - cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc)); -} - -inline static hipError_t hipDestroyTextureObject(hipTextureObject_t textureObject) { - return hipCUDAErrorTohipError(cudaDestroyTextureObject(textureObject)); -} - -inline static hipError_t hipCreateSurfaceObject(hipSurfaceObject_t* pSurfObject, - const hipResourceDesc* pResDesc) { - return hipCUDAErrorTohipError(cudaCreateSurfaceObject(pSurfObject, pResDesc)); -} - -inline static hipError_t hipDestroySurfaceObject(hipSurfaceObject_t surfaceObject) { - return hipCUDAErrorTohipError(cudaDestroySurfaceObject(surfaceObject)); -} - -inline static hipError_t hipGetTextureObjectResourceDesc(hipResourceDesc* pResDesc, - hipTextureObject_t textureObject) { - return hipCUDAErrorTohipError(cudaGetTextureObjectResourceDesc( pResDesc, textureObject)); -} - -__HIP_DEPRECATED inline static hipError_t hipGetTextureAlignmentOffset( - size_t* offset, const struct textureReference* texref) { - return hipCUDAErrorTohipError(cudaGetTextureAlignmentOffset(offset,texref)); -} - -inline static hipError_t hipGetChannelDesc(hipChannelFormatDesc* desc, hipArray_const_t array) -{ - return hipCUDAErrorTohipError(cudaGetChannelDesc(desc,array)); -} - -inline static hipError_t hipLaunchCooperativeKernel(const void* f, dim3 gridDim, dim3 blockDim, - void** kernelParams, unsigned int sharedMemBytes, - hipStream_t stream) { - return hipCUDAErrorTohipError( - cudaLaunchCooperativeKernel(f, gridDim, blockDim, kernelParams, sharedMemBytes, stream)); -} - -inline static hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList, - int numDevices, unsigned int flags) { - return hipCUDAErrorTohipError(cudaLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags)); -} - -#ifdef __cplusplus -} -#endif - -#ifdef __CUDACC__ - -template -inline static hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, - T func, - int blockSize, - size_t dynamicSMemSize) { - return hipCUDAErrorTohipError(cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, - blockSize, dynamicSMemSize)); -} - -template -inline static hipError_t hipOccupancyMaxPotentialBlockSize(int* minGridSize, int* blockSize, T func, - size_t dynamicSMemSize = 0, - int blockSizeLimit = 0) { - return hipCUDAErrorTohipError(cudaOccupancyMaxPotentialBlockSize(minGridSize, blockSize, func, - dynamicSMemSize, blockSizeLimit)); -} - -template -inline static hipError_t hipOccupancyMaxPotentialBlockSizeWithFlags(int* minGridSize, int* blockSize, T func, - size_t dynamicSMemSize = 0, - int blockSizeLimit = 0, unsigned int flags = 0) { - return hipCUDAErrorTohipError(cudaOccupancyMaxPotentialBlockSize(minGridSize, blockSize, func, - dynamicSMemSize, blockSizeLimit, flags)); -} - -template -inline static hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( int* numBlocks, T func, - int blockSize, size_t dynamicSMemSize,unsigned int flags) { - return hipCUDAErrorTohipError(cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, - blockSize, dynamicSMemSize, flags)); -} - -template -inline static hipError_t hipBindTexture(size_t* offset, const struct texture& tex, - const void* devPtr, size_t size = UINT_MAX) { - return hipCUDAErrorTohipError(cudaBindTexture(offset, tex, devPtr, size)); -} - -template -inline static hipError_t hipBindTexture(size_t* offset, struct texture& tex, - const void* devPtr, const hipChannelFormatDesc& desc, - size_t size = UINT_MAX) { - return hipCUDAErrorTohipError(cudaBindTexture(offset, tex, devPtr, desc, size)); -} - -template -__HIP_DEPRECATED inline static hipError_t hipUnbindTexture(struct texture* tex) { - return hipCUDAErrorTohipError(cudaUnbindTexture(tex)); -} - -template -__HIP_DEPRECATED inline static hipError_t hipUnbindTexture(struct texture& tex) { - return hipCUDAErrorTohipError(cudaUnbindTexture(tex)); -} - -template -__HIP_DEPRECATED inline static hipError_t hipBindTextureToArray( - struct texture& tex, hipArray_const_t array, - const hipChannelFormatDesc& desc) { - return hipCUDAErrorTohipError(cudaBindTextureToArray(tex, array, desc)); -} - -template -__HIP_DEPRECATED inline static hipError_t hipBindTextureToArray( - struct texture* tex, hipArray_const_t array, - const hipChannelFormatDesc* desc) { - return hipCUDAErrorTohipError(cudaBindTextureToArray(tex, array, desc)); -} - -template -__HIP_DEPRECATED inline static hipError_t hipBindTextureToArray( - struct texture& tex, hipArray_const_t array) { - return hipCUDAErrorTohipError(cudaBindTextureToArray(tex, array)); -} - -template -inline static hipChannelFormatDesc hipCreateChannelDesc() { - return cudaCreateChannelDesc(); -} - -template -inline static hipError_t hipLaunchCooperativeKernel(T f, dim3 gridDim, dim3 blockDim, - void** kernelParams, unsigned int sharedMemBytes, hipStream_t stream) { - return hipCUDAErrorTohipError( - cudaLaunchCooperativeKernel(reinterpret_cast(f), gridDim, blockDim, kernelParams, sharedMemBytes, stream)); -} - -inline static hipError_t hipTexRefSetAddressMode(hipTexRef hTexRef, int dim, hipAddress_mode am){ - return hipCUResultTohipError(cuTexRefSetAddressMode(hTexRef,dim,am)); -} - -inline static hipError_t hipTexRefSetFilterMode(hipTexRef hTexRef, hipFilter_mode fm){ - return hipCUResultTohipError(cuTexRefSetFilterMode(hTexRef,fm)); -} - -inline static hipError_t hipTexRefSetAddress(size_t *ByteOffset, hipTexRef hTexRef, hipDeviceptr_t dptr, size_t bytes){ - return hipCUResultTohipError(cuTexRefSetAddress(ByteOffset,hTexRef,dptr,bytes)); -} - -inline static hipError_t hipTexRefSetAddress2D(hipTexRef hTexRef, const CUDA_ARRAY_DESCRIPTOR *desc, hipDeviceptr_t dptr, size_t Pitch){ - return hipCUResultTohipError(cuTexRefSetAddress2D(hTexRef,desc,dptr,Pitch)); -} - -inline static hipError_t hipTexRefSetFormat(hipTexRef hTexRef, hipArray_Format fmt, int NumPackedComponents){ - return hipCUResultTohipError(cuTexRefSetFormat(hTexRef,fmt,NumPackedComponents)); -} - -inline static hipError_t hipTexRefSetFlags(hipTexRef hTexRef, unsigned int Flags){ - return hipCUResultTohipError(cuTexRefSetFlags(hTexRef,Flags)); -} - -inline static hipError_t hipTexRefSetArray(hipTexRef hTexRef, hiparray hArray, unsigned int Flags){ - return hipCUResultTohipError(cuTexRefSetArray(hTexRef,hArray,Flags)); -} - -inline static hipError_t hipArrayCreate(hiparray* pHandle, const HIP_ARRAY_DESCRIPTOR* pAllocateArray){ - return hipCUResultTohipError(cuArrayCreate(pHandle, pAllocateArray)); -} - -inline static hipError_t hipArrayDestroy(hiparray hArray){ - return hipCUResultTohipError(cuArrayDestroy(hArray)); -} - -inline static hipError_t hipArray3DCreate(hiparray* pHandle, - const HIP_ARRAY3D_DESCRIPTOR* pAllocateArray){ - return hipCUResultTohipError(cuArray3DCreate(pHandle, pAllocateArray)); -} - -#endif //__CUDACC__ - -#endif // HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_RUNTIME_API_H diff --git a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_texture_types.h b/src/hipamd/include/hip/nvidia_detail/nvidia_hip_texture_types.h deleted file mode 100644 index df374d705a..0000000000 --- a/src/hipamd/include/hip/nvidia_detail/nvidia_hip_texture_types.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_TEXTURE_TYPES_H -#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_TEXTURE_TYPES_H - -#include - -#endif diff --git a/src/hipamd/include/hip/nvidia_detail/nvidia_hiprtc.h b/src/hipamd/include/hip/nvidia_detail/nvidia_hiprtc.h deleted file mode 100644 index 449ba26c0f..0000000000 --- a/src/hipamd/include/hip/nvidia_detail/nvidia_hiprtc.h +++ /dev/null @@ -1,168 +0,0 @@ -/* -Copyright (c) 2021 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ -#ifndef HIPRTC_H -#define HIPRTC_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#include - -#if !defined(_WIN32) -#pragma GCC visibility push(default) -#endif - -typedef enum hiprtcResult { - HIPRTC_SUCCESS = 0, - HIPRTC_ERROR_OUT_OF_MEMORY = 1, - HIPRTC_ERROR_PROGRAM_CREATION_FAILURE = 2, - HIPRTC_ERROR_INVALID_INPUT = 3, - HIPRTC_ERROR_INVALID_PROGRAM = 4, - HIPRTC_ERROR_INVALID_OPTION = 5, - HIPRTC_ERROR_COMPILATION = 6, - HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE = 7, - HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION = 8, - HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION = 9, - HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID = 10, - HIPRTC_ERROR_INTERNAL_ERROR = 11 -} hiprtcResult; - -inline static nvrtcResult hiprtcResultTonvrtcResult(hiprtcResult result) { - switch (result) { - case HIPRTC_SUCCESS: - return NVRTC_SUCCESS; - case HIPRTC_ERROR_OUT_OF_MEMORY: - return NVRTC_ERROR_OUT_OF_MEMORY; - case HIPRTC_ERROR_PROGRAM_CREATION_FAILURE: - return NVRTC_ERROR_PROGRAM_CREATION_FAILURE; - case HIPRTC_ERROR_INVALID_INPUT: - return NVRTC_ERROR_INVALID_INPUT; - case HIPRTC_ERROR_INVALID_PROGRAM: - return NVRTC_ERROR_INVALID_PROGRAM; - case HIPRTC_ERROR_INVALID_OPTION: - return NVRTC_ERROR_INVALID_OPTION; - case HIPRTC_ERROR_COMPILATION: - return NVRTC_ERROR_COMPILATION; - case HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE: - return NVRTC_ERROR_BUILTIN_OPERATION_FAILURE; - case HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION: - return NVRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION; - case HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION: - return NVRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION; - case HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID: - return NVRTC_ERROR_NAME_EXPRESSION_NOT_VALID; - case HIPRTC_ERROR_INTERNAL_ERROR: - return NVRTC_ERROR_INTERNAL_ERROR; - } -} - -inline static hiprtcResult nvrtcResultTohiprtcResult(nvrtcResult result) { - switch (result) { - case NVRTC_SUCCESS: - return HIPRTC_SUCCESS; - case NVRTC_ERROR_OUT_OF_MEMORY: - return HIPRTC_ERROR_OUT_OF_MEMORY; - case NVRTC_ERROR_PROGRAM_CREATION_FAILURE: - return HIPRTC_ERROR_PROGRAM_CREATION_FAILURE; - case NVRTC_ERROR_INVALID_INPUT: - return HIPRTC_ERROR_INVALID_INPUT; - case NVRTC_ERROR_INVALID_PROGRAM: - return HIPRTC_ERROR_INVALID_PROGRAM; - case NVRTC_ERROR_INVALID_OPTION: - return HIPRTC_ERROR_INVALID_OPTION; - case NVRTC_ERROR_COMPILATION: - return HIPRTC_ERROR_COMPILATION; - case NVRTC_ERROR_BUILTIN_OPERATION_FAILURE: - return HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE; - case NVRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION: - return HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION; - case NVRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION: - return HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION; - case NVRTC_ERROR_NAME_EXPRESSION_NOT_VALID: - return HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID; - case NVRTC_ERROR_INTERNAL_ERROR: - return HIPRTC_ERROR_INTERNAL_ERROR; - } -} - -const char* hiprtcGetErrorString(hiprtcResult result) { - return nvrtcGetErrorString(hiprtcResultTonvrtcResult(result)); -} - -hiprtcResult hiprtcVersion(int* major, int* minor) { - return nvrtcResultTohiprtcResult(nvrtcVersion(major, minor)); -} - -typedef nvrtcProgram hiprtcProgram; - -hiprtcResult hiprtcAddNameExpression(hiprtcProgram prog, const char* name_expression) { - return nvrtcResultTohiprtcResult(nvrtcAddNameExpression(prog, name_expression)); -} - -hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, int numOptions, const char** options) { - return nvrtcResultTohiprtcResult(nvrtcCompileProgram(prog, numOptions, options)); -} - -hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, const char* src, const char* name, - int numHeaders, const char** headers, const char** includeNames) { - return nvrtcResultTohiprtcResult( - nvrtcCreateProgram(prog, src, name, numHeaders, headers, includeNames)); -} - -hiprtcResult hiprtcDestroyProgram(hiprtcProgram* prog) { - return nvrtcResultTohiprtcResult(nvrtcDestroyProgram(prog)); -} - -hiprtcResult hiprtcGetLoweredName(hiprtcProgram prog, const char* name_expression, - const char** lowered_name) { - return nvrtcResultTohiprtcResult(nvrtcGetLoweredName(prog, name_expression, lowered_name)); -} - -hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* log) { - return nvrtcResultTohiprtcResult(nvrtcGetProgramLog(prog, log)); -} - -hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog, size_t* logSizeRet) { - return nvrtcResultTohiprtcResult(nvrtcGetProgramLogSize(prog, logSizeRet)); -} - -hiprtcResult hiprtcGetCode(hiprtcProgram prog, char* code) { - return nvrtcResultTohiprtcResult(nvrtcGetPTX(prog, code)); -} - -hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, size_t* codeSizeRet) { - return nvrtcResultTohiprtcResult(nvrtcGetPTXSize(prog, codeSizeRet)); -} - -#if !defined(_WIN32) -#pragma GCC visibility pop -#endif - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif // HIPRTC_H diff --git a/src/hipamd/src/amd_hsa_elf.hpp b/src/hipamd/src/amd_hsa_elf.hpp deleted file mode 100644 index c5a7ca40a1..0000000000 --- a/src/hipamd/src/amd_hsa_elf.hpp +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright (c) 2015-2020 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -// AMDGPU OS for HSA compatible compute kernels. -enum { ELFOSABI_AMDGPU_HSA = 64, ELFOSABI_AMDGPU_PAL = 65, ELFOSABI_AMDGPU_MESA3D = 66 }; - -enum { - ELFABIVERSION_AMDGPU_HSA_V2 = 0, - ELFABIVERSION_AMDGPU_HSA_V3 = 1, - ELFABIVERSION_AMDGPU_HSA_V4 = 2 -}; - -// AMDGPU specific e_flags -enum : unsigned { - EF_AMDGPU_MACH = 0x0ff, - // AMDGPU processors - EF_AMDGPU_MACH_NONE = 0x000, - EF_AMDGPU_MACH_R600_R600 = 0x001, - EF_AMDGPU_MACH_R600_R630 = 0x002, - EF_AMDGPU_MACH_R600_RS880 = 0x003, - EF_AMDGPU_MACH_R600_RV670 = 0x004, - EF_AMDGPU_MACH_R600_RV710 = 0x005, - EF_AMDGPU_MACH_R600_RV730 = 0x006, - EF_AMDGPU_MACH_R600_RV770 = 0x007, - EF_AMDGPU_MACH_R600_CEDAR = 0x008, - EF_AMDGPU_MACH_R600_CYPRESS = 0x009, - EF_AMDGPU_MACH_R600_JUNIPER = 0x00a, - EF_AMDGPU_MACH_R600_REDWOOD = 0x00b, - EF_AMDGPU_MACH_R600_SUMO = 0x00c, - EF_AMDGPU_MACH_R600_BARTS = 0x00d, - EF_AMDGPU_MACH_R600_CAICOS = 0x00e, - EF_AMDGPU_MACH_R600_CAYMAN = 0x00f, - EF_AMDGPU_MACH_R600_TURKS = 0x010, - EF_AMDGPU_MACH_R600_RESERVED_FIRST = 0x011, - EF_AMDGPU_MACH_R600_RESERVED_LAST = 0x01f, - EF_AMDGPU_MACH_R600_FIRST = EF_AMDGPU_MACH_R600_R600, - EF_AMDGPU_MACH_R600_LAST = EF_AMDGPU_MACH_R600_TURKS, - - EF_AMDGPU_MACH_AMDGCN_GFX600 = 0x020, - EF_AMDGPU_MACH_AMDGCN_GFX601 = 0x021, - EF_AMDGPU_MACH_AMDGCN_GFX700 = 0x022, - EF_AMDGPU_MACH_AMDGCN_GFX701 = 0x023, - EF_AMDGPU_MACH_AMDGCN_GFX702 = 0x024, - EF_AMDGPU_MACH_AMDGCN_GFX703 = 0x025, - EF_AMDGPU_MACH_AMDGCN_GFX704 = 0x026, - EF_AMDGPU_MACH_AMDGCN_RESERVED_0X027 = 0x027, - EF_AMDGPU_MACH_AMDGCN_GFX801 = 0x028, - EF_AMDGPU_MACH_AMDGCN_GFX802 = 0x029, - EF_AMDGPU_MACH_AMDGCN_GFX803 = 0x02a, - EF_AMDGPU_MACH_AMDGCN_GFX810 = 0x02b, - EF_AMDGPU_MACH_AMDGCN_GFX900 = 0x02c, - EF_AMDGPU_MACH_AMDGCN_GFX902 = 0x02d, - EF_AMDGPU_MACH_AMDGCN_GFX904 = 0x02e, - EF_AMDGPU_MACH_AMDGCN_GFX906 = 0x02f, - EF_AMDGPU_MACH_AMDGCN_GFX908 = 0x030, - EF_AMDGPU_MACH_AMDGCN_GFX909 = 0x031, - EF_AMDGPU_MACH_AMDGCN_GFX90C = 0x032, - EF_AMDGPU_MACH_AMDGCN_GFX1010 = 0x033, - EF_AMDGPU_MACH_AMDGCN_GFX1011 = 0x034, - EF_AMDGPU_MACH_AMDGCN_GFX1012 = 0x035, - EF_AMDGPU_MACH_AMDGCN_GFX1030 = 0x036, - EF_AMDGPU_MACH_AMDGCN_GFX1031 = 0x037, - EF_AMDGPU_MACH_AMDGCN_GFX1032 = 0x038, - EF_AMDGPU_MACH_AMDGCN_GFX1033 = 0x039, - EF_AMDGPU_MACH_AMDGCN_GFX602 = 0x03a, - EF_AMDGPU_MACH_AMDGCN_GFX705 = 0x03b, - EF_AMDGPU_MACH_AMDGCN_GFX805 = 0x03c, - EF_AMDGPU_MACH_AMDGCN_GFX90A = 0x03f, - EF_AMDGPU_MACH_AMDGCN_FIRST = EF_AMDGPU_MACH_AMDGCN_GFX600, - EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX90A, - - // Indicates if the "xnack" target feature is enabled for all code contained - // in the object. - // - // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V3. - EF_AMDGPU_FEATURE_XNACK_V3 = 0x100, - // Indicates if the "sramecc" target feature is enabled for all code - // contained in the object. - // - // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V3. - EF_AMDGPU_FEATURE_SRAMECC_V3 = 0x200, - - // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V4. - EF_AMDGPU_FEATURE_XNACK_V4 = 0x300, - EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4 = 0x000, - EF_AMDGPU_FEATURE_XNACK_ANY_V4 = 0x100, - EF_AMDGPU_FEATURE_XNACK_OFF_V4 = 0x200, - EF_AMDGPU_FEATURE_XNACK_ON_V4 = 0x300, - - // SRAMECC selection mask for EF_AMDGPU_FEATURE_SRAMECC_* values. - // Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V4. - EF_AMDGPU_FEATURE_SRAMECC_V4 = 0xc00, - EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4 = 0x000, - EF_AMDGPU_FEATURE_SRAMECC_ANY_V4 = 0x400, - EF_AMDGPU_FEATURE_SRAMECC_OFF_V4 = 0x800, - EF_AMDGPU_FEATURE_SRAMECC_ON_V4 = 0xc00, -}; diff --git a/src/hipamd/src/amdhip.def b/src/hipamd/src/amdhip.def deleted file mode 100644 index 559a08289d..0000000000 --- a/src/hipamd/src/amdhip.def +++ /dev/null @@ -1,293 +0,0 @@ -EXPORTS -hipChooseDevice -hipCtxCreate -hipCtxDestroy -hipCtxDisablePeerAccess -hipCtxEnablePeerAccess -hipCtxGetApiVersion -hipCtxGetCacheConfig -hipCtxGetCurrent -hipCtxGetDevice -hipCtxGetFlags -hipCtxGetSharedMemConfig -hipCtxPopCurrent -hipCtxPushCurrent -hipCtxSetCacheConfig -hipCtxSetCurrent -hipCtxSetSharedMemConfig -hipCtxSynchronize -hipDeviceCanAccessPeer -hipDeviceComputeCapability -hipDeviceDisablePeerAccess -hipDeviceEnablePeerAccess -hipDeviceGet -hipDeviceGetAttribute -hipDeviceGetByPCIBusId -hipDeviceGetCacheConfig -hipDeviceGetStreamPriorityRange -hipDeviceGetLimit -hipDeviceGetName -hipDeviceGetPCIBusId -hipDeviceGetSharedMemConfig -hipDeviceGetP2PAttribute -hipDevicePrimaryCtxGetState -hipDevicePrimaryCtxRelease -hipDevicePrimaryCtxReset -hipDevicePrimaryCtxRetain -hipDevicePrimaryCtxSetFlags -hipDeviceReset -hipDeviceSetCacheConfig -hipDeviceSetSharedMemConfig -hipDeviceSynchronize -hipDeviceTotalMem -hipDriverGetVersion -hipEventCreate -hipEventCreateWithFlags -hipEventDestroy -hipEventElapsedTime -hipEventQuery -hipEventRecord -hipEventSynchronize -hipExtGetLinkTypeAndHopCount -hipExtLaunchMultiKernelMultiDevice -hipExtMallocWithFlags -hipExtModuleLaunchKernel -hipExtLaunchKernel -hipFree -hipFreeArray -hipFuncSetAttribute -hipFuncSetCacheConfig -hipFuncSetSharedMemConfig -hipGetDevice -hipGetDeviceCount -hipGetDeviceProperties -hipGetErrorName -hipGetErrorString -hipGetLastError -hipMemAllocHost -hipHostAlloc -hipHostFree -hipHostGetDevicePointer -hipHostGetFlags -hipHostMalloc -hipHostRegister -hipHostUnregister -hipInit -hipIpcCloseMemHandle -hipIpcGetMemHandle -hipIpcOpenMemHandle -hipIpcGetEventHandle -hipIpcOpenEventHandle -hipMalloc -hipMalloc3D -hipMalloc3DArray -hipMallocManaged -hipArrayCreate -hipArray3DCreate -hipArrayDestroy -hipMallocArray -hipMemAdvise -hipMemAllocPitch -hipMallocPitch -hipMemcpy -hipMemcpyWithStream -hipMemcpyParam2D -hipMemcpy2D -hipMemcpy2DAsync -hipMemcpy2DToArray -hipMemcpy2DToArrayAsync -hipMemcpy3D -hipMemcpy3DAsync -hipDrvMemcpy3D -hipDrvMemcpy3DAsync -hipMemcpyAsync -hipMemcpyDtoD -hipMemcpyDtoDAsync -hipMemcpyDtoH -hipMemcpyDtoHAsync -hipMemcpyFromSymbol -hipMemcpyFromSymbolAsync -hipMemcpyHtoD -hipMemcpyHtoDAsync -hipMemcpyPeer -hipMemcpyPeerAsync -hipMemcpyToArray -hipMemcpyFromArray -hipMemcpyToSymbol -hipMemcpyToSymbolAsync -hipMemGetAddressRange -hipGetSymbolAddress -hipGetSymbolSize -hipMemGetInfo -hipMemPrefetchAsync -hipMemPtrGetInfo -hipMemRangeGetAttribute -hipMemRangeGetAttributes -hipMemset -hipMemsetAsync -hipMemsetD8 -hipMemsetD8Async -hipMemsetD16 -hipMemsetD16Async -hipMemsetD32 -hipMemsetD32Async -hipMemset2D -hipMemset2DAsync -hipMemset3D -hipMemset3DAsync -hipModuleGetFunction -hipModuleGetGlobal -hipModuleGetTexRef -hipModuleLaunchKernel -hipModuleLaunchKernelExt -hipLaunchCooperativeKernel -hipLaunchCooperativeKernelMultiDevice -hipHccModuleLaunchKernel -hipModuleLoad -hipModuleLoadData -hipModuleLoadDataEx -hipModuleUnload -hipModuleOccupancyMaxPotentialBlockSize -hipModuleOccupancyMaxPotentialBlockSizeWithFlags -hipModuleOccupancyMaxActiveBlocksPerMultiprocessor -hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags -hipOccupancyMaxPotentialBlockSize -hipOccupancyMaxActiveBlocksPerMultiprocessor -hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags -hipFuncGetAttribute -hipFuncGetAttributes -hipPeekAtLastError -hipPointerGetAttributes -hipProfilerStart -hipProfilerStop -hipRuntimeGetVersion -hipGetDeviceFlags -hipSetDevice -hipSetDeviceFlags -hipStreamAddCallback -hipStreamAttachMemAsync -hipStreamCreate -hipStreamCreateWithFlags -hipStreamCreateWithPriority -hipStreamDestroy -hipStreamGetFlags -hipStreamQuery -hipStreamSynchronize -hipStreamWaitEvent -__hipPopCallConfiguration -__hipPushCallConfiguration -__hipRegisterFatBinary -__hipRegisterFunction -__hipRegisterVar -__hipRegisterSurface -__hipRegisterTexture -__hipRegisterManagedVar -__hipUnregisterFatBinary -hipConfigureCall -hipSetupArgument -hipLaunchByPtr -hipLaunchKernel -hipRegisterApiCallback -hipRemoveApiCallback -hipRegisterActivityCallback -hipRemoveActivityCallback -hipApiName -hipKernelNameRef -hipBindTexture -hipBindTexture2D -hipBindTextureToArray -hipBindTextureToMipmappedArray -hipGetTextureAlignmentOffset -hipGetTextureReference -hipUnbindTexture -hipCreateChannelDesc -hipCreateTextureObject -hipDestroyTextureObject -hipGetChannelDesc -hipGetTextureObjectResourceDesc -hipGetTextureObjectResourceViewDesc -hipGetTextureObjectTextureDesc -hipTexRefGetAddress -hipTexRefGetAddressMode -hipTexRefGetArray -hipTexRefGetBorderColor -hipTexRefGetFilterMode -hipTexRefGetFlags -hipTexRefGetFormat -hipTexRefGetMaxAnisotropy -hipTexRefGetMipmapFilterMode -hipTexRefGetMipmapLevelBias -hipTexRefGetMipmapLevelClamp -hipTexRefGetMipmappedArray -hipTexRefSetAddress -hipTexRefSetAddress2D -hipTexRefSetAddressMode -hipTexRefSetArray -hipTexRefSetBorderColor -hipTexRefSetFilterMode -hipTexRefSetFlags -hipTexRefSetFormat -hipTexRefSetMaxAnisotropy -hipTexRefSetMipmapFilterMode -hipTexRefSetMipmapLevelBias -hipTexRefSetMipmapLevelClamp -hipTexRefSetMipmappedArray -hipProfilerStart -hipProfilerStop -hipCreateSurfaceObject -hipDestroySurfaceObject -hipInitActivityCallback -hipEnableActivityCallback -hipGetCmdName -hiprtcAddNameExpression -hiprtcCompileProgram -hiprtcCreateProgram -hiprtcDestroyProgram -hiprtcGetLoweredName -hiprtcGetProgramLog -hiprtcGetProgramLogSize -hiprtcGetCode -hiprtcGetCodeSize -hiprtcGetErrorString -hipMipmappedArrayCreate -hipMallocMipmappedArray -hipMipmappedArrayDestroy -hipFreeMipmappedArray -hipMipmappedArrayGetLevel -hipGetMipmappedArrayLevel -hipMallocHost -hipFreeHost -hipTexObjectCreate -hipTexObjectDestroy -hipTexObjectGetResourceDesc -hipTexObjectGetResourceViewDesc -hipTexObjectGetTextureDesc -hipExtStreamCreateWithCUMask -hipStreamGetPriority -hipMemcpy2DFromArray -hipMemcpy2DFromArrayAsync -hipDrvMemcpy2DUnaligned -hipMemcpyAtoH -hipMemcpyHtoA -hipMemcpyParam2DAsync -__gnu_h2f_ieee -__gnu_f2h_ieee -hipExtStreamGetCUMask -hipImportExternalMemory -hipExternalMemoryGetMappedBuffer -hipDestroyExternalMemory -hipGraphCreate -hipGraphDestroy -hipGraphAddKernelNode -hipGraphAddMemsetNode -hipGraphAddMemcpyNode -hipGraphInstantiate -hipGraphLaunch -hipStreamIsCapturing -hipStreamBeginCapture -hipStreamEndCapture -hipGraphExecDestroy -hipImportExternalSemaphore -hipSignalExternalSemaphoresAsync -hipWaitExternalSemaphoresAsync -hipDestroyExternalSemaphore diff --git a/src/hipamd/src/cl_d3d10.cpp b/src/hipamd/src/cl_d3d10.cpp deleted file mode 100644 index 0c0dcdd5b8..0000000000 --- a/src/hipamd/src/cl_d3d10.cpp +++ /dev/null @@ -1,1451 +0,0 @@ -/* Copyright (c) 2009-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifdef _WIN32 - -#include "top.hpp" - -#include "cl_common.hpp" -#include "cl_d3d10_amd.hpp" -#include "platform/command.hpp" - -#include -#include - - -/*! \addtogroup API - * @{ - * - * \addtogroup CL_D3D10_Interops - * - * This section discusses OpenCL functions that allow applications to use Direct3D 10 - * resources (buffers/textures) as OpenCL memory objects. This allows efficient sharing of - * data between OpenCL and Direct3D 10. The OpenCL API can be used to execute kernels that - * read and/or write memory objects that are also the Direct3D resources. - * An OpenCL image object can be created from a D3D10 texture object. An - * OpenCL buffer object can be created from a D3D10 buffer object (index/vertex). - * - * @} - * \addtogroup clGetDeviceIDsFromD3D10KHR - * @{ - */ - -RUNTIME_ENTRY(cl_int, clGetDeviceIDsFromD3D10KHR, - (cl_platform_id platform, cl_d3d10_device_source_khr d3d_device_source, - void* d3d_object, cl_d3d10_device_set_khr d3d_device_set, cl_uint num_entries, - cl_device_id* devices, cl_uint* num_devices)) { - cl_int errcode; - ID3D10Device* d3d10_device = NULL; - cl_device_id* gpu_devices; - cl_uint num_gpu_devices = 0; - bool create_d3d10Device = false; - static const bool VALIDATE_ONLY = true; - HMODULE d3d10Module = NULL; - - if (platform != NULL && platform != AMD_PLATFORM) { - LogWarning("\"platrform\" is not a valid AMD platform"); - return CL_INVALID_PLATFORM; - } - if (((num_entries > 0 || num_devices == NULL) && devices == NULL) || - (num_entries == 0 && devices != NULL)) { - return CL_INVALID_VALUE; - } - // Get GPU devices - errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 0, NULL, &num_gpu_devices); - if (errcode != CL_SUCCESS && errcode != CL_DEVICE_NOT_FOUND) { - return CL_INVALID_VALUE; - } - - if (!num_gpu_devices) { - *not_null(num_devices) = 0; - return CL_DEVICE_NOT_FOUND; - } - - switch (d3d_device_source) { - case CL_D3D10_DEVICE_KHR: - d3d10_device = static_cast(d3d_object); - break; - case CL_D3D10_DXGI_ADAPTER_KHR: { - typedef HRESULT(WINAPI * LPD3D10CREATEDEVICE)(IDXGIAdapter*, D3D10_DRIVER_TYPE, HMODULE, UINT, - UINT32, ID3D10Device**); - static LPD3D10CREATEDEVICE dynamicD3D10CreateDevice = NULL; - - d3d10Module = LoadLibrary("D3D10.dll"); - if (d3d10Module == NULL) { - return CL_INVALID_PLATFORM; - } - - dynamicD3D10CreateDevice = - (LPD3D10CREATEDEVICE)GetProcAddress(d3d10Module, "D3D10CreateDevice"); - - IDXGIAdapter* dxgi_adapter = static_cast(d3d_object); - HRESULT hr = dynamicD3D10CreateDevice(dxgi_adapter, D3D10_DRIVER_TYPE_HARDWARE, NULL, 0, - D3D10_SDK_VERSION, &d3d10_device); - if (SUCCEEDED(hr) && (NULL != d3d10_device)) { - create_d3d10Device = true; - } else { - FreeLibrary(d3d10Module); - return CL_INVALID_VALUE; - } - } break; - default: - LogWarning("\"d3d_device_source\" is invalid"); - return CL_INVALID_VALUE; - } - - switch (d3d_device_set) { - case CL_PREFERRED_DEVICES_FOR_D3D10_KHR: - case CL_ALL_DEVICES_FOR_D3D10_KHR: { - gpu_devices = (cl_device_id*)alloca(num_gpu_devices * sizeof(cl_device_id)); - - errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, num_gpu_devices, gpu_devices, NULL); - if (errcode != CL_SUCCESS) { - break; - } - - void* external_device[amd::Context::DeviceFlagIdx::LastDeviceFlagIdx] = {}; - external_device[amd::Context::DeviceFlagIdx::D3D10DeviceKhrIdx] = d3d10_device; - - std::vector compatible_devices; - for (cl_uint i = 0; i < num_gpu_devices; ++i) { - cl_device_id device = gpu_devices[i]; - if (is_valid(device) && - as_amd(device)->bindExternalDevice(amd::Context::Flags::D3D10DeviceKhr, external_device, - NULL, VALIDATE_ONLY)) { - compatible_devices.push_back(as_amd(device)); - } - } - if (compatible_devices.size() == 0) { - *not_null(num_devices) = 0; - errcode = CL_DEVICE_NOT_FOUND; - break; - } - - auto it = compatible_devices.cbegin(); - cl_uint compatible_count = std::min(num_entries, (cl_uint)compatible_devices.size()); - - while (compatible_count--) { - *devices++ = as_cl(*it++); - --num_entries; - } - while (num_entries--) { - *devices++ = (cl_device_id)0; - } - - *not_null(num_devices) = (cl_uint)compatible_devices.size(); - } break; - - default: - LogWarning("\"d3d_device_set\" is invalid"); - errcode = CL_INVALID_VALUE; - } - - if (create_d3d10Device) { - d3d10_device->Release(); - FreeLibrary(d3d10Module); - } - return errcode; -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromD3D10BufferKHR - * @{ - */ - -/*! \brief Creates an OpenCL buffer object from a Direct3D 10 resource. - * - * \param context is a valid OpenCL context. - * - * \param flags is a bit-field that is used to specify usage information. - * Only CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE values - * can be used. - * - * \param pD3DResource is a valid pointer to a D3D10 resource of type ID3D10Buffer. - * - * \return valid non-zero OpenCL buffer object and \a errcode_ret is set - * to CL_SUCCESS if the buffer object is created successfully. It returns a NULL - * value with one of the following error values returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a context is not a valid context or if Direct3D 10 - * interoperatbility has not been initialized between context and the ID3D10Device - * from which pD3DResource was created. - * - CL_INVALID_VALUE if values specified in \a clFlags are not valid. - * - CL_INVALID_D3D_RESOURCE if \a pD3DResource is not of type ID3D10Buffer. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.0r33? - */ - -RUNTIME_ENTRY_RET(cl_mem, clCreateFromD3D10BufferKHR, - (cl_context context, cl_mem_flags flags, ID3D10Buffer* pD3DResource, - cl_int* errcode_ret)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - if (!flags) flags = CL_MEM_READ_WRITE; - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - if (!pD3DResource) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("parameter \"pD3DResource\" is a NULL pointer"); - return clMemObj; - } - return ( - amd::clCreateBufferFromD3D10ResourceAMD(*as_amd(context), flags, pD3DResource, errcode_ret)); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateImageFromD3D10Resource - * @{ - */ - -/*! \brief Create an OpenCL 2D or 3D image object from a D3D10 resource. - * - * \param context is a valid OpenCL context. - * - * \param flags is a bit-field that is used to specify usage information. - * Only CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE values - * can be used. - * - * \param pD3DResource is a valid pointer to a D3D10 resource of type - * ID3D10Texture2D, ID3D10Texture2D, or ID3D10Texture3D. - * If pD3DResource is of type ID3D10Texture1D then the created image object - * will be a 1D mipmapped image object. - * If pD3DResource is of type ID3D10Texture2D and was not created with flag - * D3D10_RESOURCE_MISC_TEXTURECUBE then the created image object will be a - * 2D mipmapped image object. - * If pD3DResource is of type ID3D10Texture2D and was created with flag - * D3D10_RESOURCE_MISC_TEXTURECUBE then the created image object will be - * a cubemap mipmapped image object. - * errocde_ret returns CL_INVALID_D3D_RESOURCE if an OpenCL memory object has - * already been created from pD3DResource in context. - * If pD3DResource is of type ID3D10Texture3D then the created image object will - * be a 3D mipmapped imageobject. - * - * \return valid non-zero OpenCL image object and \a errcode_ret is set - * to CL_SUCCESS if the image object is created successfully. It returns a NULL - * value with one of the following error values returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a context is not a valid context or if Direct3D 10 - * interoperatbility has not been initialized between context and the ID3D10Device - * from which pD3DResource was created. - * - CL_INVALID_VALUE if values specified in \a flags are not valid. - * - CL_INVALID_D3D_RESOURCE if \a pD3DResource is not of type ID3D10Texture1D, - * ID3D10Texture2D, or ID3D10Texture3D. - * - CL_INVALID_D3D_RESOURCE if an OpenCL memory object has already been created - * from \a pD3DResource in context. - * - CL_INVALID_IMAGE_FORMAT if the Direct3D 10 texture format does not map - * to an appropriate OpenCL image format. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.0r48? - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateImageFromD3D10Resource, - (cl_context context, cl_mem_flags flags, ID3D10Resource* pD3DResource, - UINT subresource, int* errcode_ret, UINT dimension)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - if (!flags) flags = CL_MEM_READ_WRITE; - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - if (!pD3DResource) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("parameter \"pD3DResource\" is a NULL pointer"); - return clMemObj; - } - - // Verify context init'ed for interop - ID3D10Device* pDev; - pD3DResource->GetDevice(&pDev); - if (pDev == NULL) { - *not_null(errcode_ret) = CL_INVALID_D3D10_DEVICE_KHR; - LogWarning("Cannot retrieve D3D10 device from D3D10 resource"); - return (cl_mem)0; - } - pDev->Release(); - if (!((*as_amd(context)).info().flags_ & amd::Context::D3D10DeviceKhr)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("\"amdContext\" is not created from D3D10 device"); - return (cl_mem)0; - } - - // Check for image support - const std::vector& devices = as_amd(context)->devices(); - bool supportPass = false; - bool sizePass = false; - for (const auto& it : devices) { - if (it->info().imageSupport_) { - supportPass = true; - } - } - if (!supportPass) { - *not_null(errcode_ret) = CL_INVALID_OPERATION; - LogWarning("there are no devices in context to support images"); - return (cl_mem)0; - } - - switch (dimension) { -#if 0 - case 1: - return(amd::clCreateImage1DFromD3D10ResourceAMD( - *as_amd(context), - flags, - pD3DResource, - subresource, - errcode_ret)); -#endif // 0 - case 2: - return (amd::clCreateImage2DFromD3D10ResourceAMD(*as_amd(context), flags, pD3DResource, - subresource, errcode_ret)); - case 3: - return (amd::clCreateImage3DFromD3D10ResourceAMD(*as_amd(context), flags, pD3DResource, - subresource, errcode_ret)); - default: - break; - } - - *not_null(errcode_ret) = CL_INVALID_D3D10_RESOURCE_KHR; - return (cl_mem)0; -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromD3D10Texture2DKHR - * @{ - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromD3D10Texture2DKHR, - (cl_context context, cl_mem_flags flags, ID3D10Texture2D* resource, - UINT subresource, cl_int* errcode_ret)) { - return clCreateImageFromD3D10Resource(context, flags, resource, subresource, errcode_ret, 2); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromD3D10Texture3DKHR - * @{ - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromD3D10Texture3DKHR, - (cl_context context, cl_mem_flags flags, ID3D10Texture3D* resource, - UINT subresource, cl_int* errcode_ret)) { - return clCreateImageFromD3D10Resource(context, flags, resource, subresource, errcode_ret, 3); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clEnqueueAcquireD3D10ObjectsKHR - * @{ - */ -RUNTIME_ENTRY(cl_int, clEnqueueAcquireD3D10ObjectsKHR, - (cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return amd::clEnqueueAcquireExtObjectsAMD(command_queue, num_objects, mem_objects, - num_events_in_wait_list, event_wait_list, event, - CL_COMMAND_ACQUIRE_D3D10_OBJECTS_KHR); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clEnqueueReleaseD3D10ObjectsKHR - * @{ - */ -RUNTIME_ENTRY(cl_int, clEnqueueReleaseD3D10ObjectsKHR, - (cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return amd::clEnqueueReleaseExtObjectsAMD(command_queue, num_objects, mem_objects, - num_events_in_wait_list, event_wait_list, event, - CL_COMMAND_RELEASE_D3D10_OBJECTS_KHR); -} -RUNTIME_EXIT - - -// -// -// namespace amd -// -// -namespace amd { -/*! @} - * \addtogroup CL-D3D10 interop helper functions - * @{ - */ - - -//******************************************************************* -// -// Internal implementation of CL API functions -// -//******************************************************************* -// -// clCreateBufferFromD3D10ResourceAMD -// -cl_mem clCreateBufferFromD3D10ResourceAMD(Context& amdContext, cl_mem_flags flags, - ID3D10Resource* pD3DResource, int* errcode_ret) { - // Verify pD3DResource is a buffer - D3D10_RESOURCE_DIMENSION rType; - pD3DResource->GetType(&rType); - if (rType != D3D10_RESOURCE_DIMENSION_BUFFER) { - *not_null(errcode_ret) = CL_INVALID_D3D10_RESOURCE_KHR; - return (cl_mem)0; - } - - D3D10Object obj; - int errcode = D3D10Object::initD3D10Object(amdContext, pD3DResource, 0, obj); - if (CL_SUCCESS != errcode) { - *not_null(errcode_ret) = errcode; - return (cl_mem)0; - } - - BufferD3D10* pBufferD3D10 = new (amdContext) BufferD3D10(amdContext, flags, obj); - if (!pBufferD3D10) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem)0; - } - if (!pBufferD3D10->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pBufferD3D10->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pBufferD3D10); -} -#if 0 -// There is no support for 1D images in the base imagee code -// -// clCreateImage1DFromD3D10ResourceAMD -// -cl_mem clCreateImage1DFromD3D10ResourceAMD( - Context& amdContext, - cl_mem_flags flags, - ID3D10Resource* pD3DResource, - UINT subresource, - int* errcode_ret) -{ - - // Verify the resource is a 1D texture - D3D10_RESOURCE_DIMENSION rType; - pD3DResource->GetType(&rType); - if(rType != D3D10_RESOURCE_DIMENSION_TEXTURE1D) { - *not_null(errcode_ret) = CL_INVALID_D3D10_RESOURCE_KHR; - return (cl_mem) 0; - } - - D3D10Object obj; - int errcode = D3D10Object::initD3D10Object(pD3DResource, subresource, obj); - if(CL_SUCCESS != errcode) - { - *not_null(errcode_ret) = errcode; - return (cl_mem) 0; - } - - Image1DD3D10 *pImage1DD3D10 = new Image1DD3D10(amdContext, flags, obj); - if(!pImage1DD3D10) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem) 0; - } - if (!pImage1DD3D10->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImage1DD3D10->release(); - return (cl_mem) 0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImage1DD3D10); -} -#endif - -// -// clCreateImage2DFromD3D10ResourceAMD -// -cl_mem clCreateImage2DFromD3D10ResourceAMD(Context& amdContext, cl_mem_flags flags, - ID3D10Resource* pD3DResource, UINT subresource, - int* errcode_ret) { - // Verify the resource is a 2D texture - D3D10_RESOURCE_DIMENSION rType; - pD3DResource->GetType(&rType); - if (rType != D3D10_RESOURCE_DIMENSION_TEXTURE2D) { - *not_null(errcode_ret) = CL_INVALID_D3D10_RESOURCE_KHR; - return (cl_mem)0; - } - - D3D10Object obj; - int errcode = D3D10Object::initD3D10Object(amdContext, pD3DResource, subresource, obj); - if (CL_SUCCESS != errcode) { - *not_null(errcode_ret) = errcode; - return (cl_mem)0; - } - - Image2DD3D10* pImage2DD3D10 = new (amdContext) Image2DD3D10(amdContext, flags, obj); - if (!pImage2DD3D10) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem)0; - } - if (!pImage2DD3D10->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImage2DD3D10->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImage2DD3D10); -} - -// -// clCreateImage2DFromD3D10ResourceAMD -// -cl_mem clCreateImage3DFromD3D10ResourceAMD(Context& amdContext, cl_mem_flags flags, - ID3D10Resource* pD3DResource, UINT subresource, - int* errcode_ret) { - // Verify the resource is a 2D texture - D3D10_RESOURCE_DIMENSION rType; - pD3DResource->GetType(&rType); - if (rType != D3D10_RESOURCE_DIMENSION_TEXTURE3D) { - *not_null(errcode_ret) = CL_INVALID_D3D10_RESOURCE_KHR; - return (cl_mem)0; - } - - D3D10Object obj; - int errcode = D3D10Object::initD3D10Object(amdContext, pD3DResource, subresource, obj); - if (CL_SUCCESS != errcode) { - *not_null(errcode_ret) = errcode; - return (cl_mem)0; - } - - Image3DD3D10* pImage3DD3D10 = new (amdContext) Image3DD3D10(amdContext, flags, obj); - if (!pImage3DD3D10) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem)0; - } - if (!pImage3DD3D10->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImage3DD3D10->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImage3DD3D10); -} - -// -// Helper function SyncD3D10Objects -// -void SyncD3D10Objects(std::vector& memObjects) { - Memory*& mem = memObjects.front(); - if (!mem) { - LogWarning("\nNULL memory object\n"); - return; - } - InteropObject* interop = mem->getInteropObj(); - if (!interop) { - LogWarning("\nNULL interop object\n"); - return; - } - D3D10Object* d3d10Obj = interop->asD3D10Object(); - if (!d3d10Obj) { - LogWarning("\nNULL D3D10 object\n"); - return; - } - ID3D10Query* query = d3d10Obj->getQuery(); - if (!query) { - LogWarning("\nNULL ID3D10Query\n"); - return; - } - query->End(); - BOOL data = FALSE; - while (S_OK != query->GetData(&data, sizeof(BOOL), 0)) { - } -} - -// -// Class D3D10Object implementation -// -size_t D3D10Object::getElementBytes(DXGI_FORMAT dxgiFmt) { - size_t bytesPerPixel; - - switch (dxgiFmt) { - case DXGI_FORMAT_R32G32B32A32_TYPELESS: - case DXGI_FORMAT_R32G32B32A32_FLOAT: - case DXGI_FORMAT_R32G32B32A32_UINT: - case DXGI_FORMAT_R32G32B32A32_SINT: - bytesPerPixel = 16; - break; - - case DXGI_FORMAT_R32G32B32_TYPELESS: - case DXGI_FORMAT_R32G32B32_FLOAT: - case DXGI_FORMAT_R32G32B32_UINT: - case DXGI_FORMAT_R32G32B32_SINT: - bytesPerPixel = 12; - break; - - case DXGI_FORMAT_R16G16B16A16_TYPELESS: - case DXGI_FORMAT_R16G16B16A16_FLOAT: - case DXGI_FORMAT_R16G16B16A16_UNORM: - case DXGI_FORMAT_R16G16B16A16_UINT: - case DXGI_FORMAT_R16G16B16A16_SNORM: - case DXGI_FORMAT_R16G16B16A16_SINT: - case DXGI_FORMAT_R32G32_TYPELESS: - case DXGI_FORMAT_R32G32_FLOAT: - case DXGI_FORMAT_R32G32_UINT: - case DXGI_FORMAT_R32G32_SINT: - case DXGI_FORMAT_R32G8X24_TYPELESS: - case DXGI_FORMAT_D32_FLOAT_S8X24_UINT: - case DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS: - case DXGI_FORMAT_X32_TYPELESS_G8X24_UINT: - bytesPerPixel = 8; - break; - - case DXGI_FORMAT_R10G10B10A2_TYPELESS: - case DXGI_FORMAT_R10G10B10A2_UNORM: - case DXGI_FORMAT_R10G10B10A2_UINT: - case DXGI_FORMAT_R11G11B10_FLOAT: - case DXGI_FORMAT_R8G8B8A8_TYPELESS: - case DXGI_FORMAT_R8G8B8A8_UNORM: - case DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: - case DXGI_FORMAT_R8G8B8A8_UINT: - case DXGI_FORMAT_R8G8B8A8_SNORM: - case DXGI_FORMAT_R8G8B8A8_SINT: - case DXGI_FORMAT_R16G16_TYPELESS: - case DXGI_FORMAT_R16G16_FLOAT: - case DXGI_FORMAT_R16G16_UNORM: - case DXGI_FORMAT_R16G16_UINT: - case DXGI_FORMAT_R16G16_SNORM: - case DXGI_FORMAT_R16G16_SINT: - case DXGI_FORMAT_R32_TYPELESS: - case DXGI_FORMAT_D32_FLOAT: - case DXGI_FORMAT_R32_FLOAT: - case DXGI_FORMAT_R32_UINT: - case DXGI_FORMAT_R32_SINT: - case DXGI_FORMAT_R24G8_TYPELESS: - case DXGI_FORMAT_D24_UNORM_S8_UINT: - case DXGI_FORMAT_R24_UNORM_X8_TYPELESS: - case DXGI_FORMAT_X24_TYPELESS_G8_UINT: - - case DXGI_FORMAT_R9G9B9E5_SHAREDEXP: - case DXGI_FORMAT_R8G8_B8G8_UNORM: - case DXGI_FORMAT_G8R8_G8B8_UNORM: - - case DXGI_FORMAT_B8G8R8A8_UNORM: - case DXGI_FORMAT_B8G8R8X8_UNORM: - bytesPerPixel = 4; - break; - - case DXGI_FORMAT_R8G8_TYPELESS: - case DXGI_FORMAT_R8G8_UNORM: - case DXGI_FORMAT_R8G8_UINT: - case DXGI_FORMAT_R8G8_SNORM: - case DXGI_FORMAT_R8G8_SINT: - case DXGI_FORMAT_R16_TYPELESS: - case DXGI_FORMAT_R16_FLOAT: - case DXGI_FORMAT_D16_UNORM: - case DXGI_FORMAT_R16_UNORM: - case DXGI_FORMAT_R16_UINT: - case DXGI_FORMAT_R16_SNORM: - case DXGI_FORMAT_R16_SINT: - - case DXGI_FORMAT_B5G6R5_UNORM: - case DXGI_FORMAT_B5G5R5A1_UNORM: - bytesPerPixel = 2; - break; - - case DXGI_FORMAT_R8_TYPELESS: - case DXGI_FORMAT_R8_UNORM: - case DXGI_FORMAT_R8_UINT: - case DXGI_FORMAT_R8_SNORM: - case DXGI_FORMAT_R8_SINT: - case DXGI_FORMAT_A8_UNORM: - case DXGI_FORMAT_R1_UNORM: - bytesPerPixel = 1; - break; - - - case DXGI_FORMAT_BC1_TYPELESS: - case DXGI_FORMAT_BC1_UNORM: - case DXGI_FORMAT_BC1_UNORM_SRGB: - case DXGI_FORMAT_BC2_TYPELESS: - case DXGI_FORMAT_BC2_UNORM: - case DXGI_FORMAT_BC2_UNORM_SRGB: - case DXGI_FORMAT_BC3_TYPELESS: - case DXGI_FORMAT_BC3_UNORM: - case DXGI_FORMAT_BC3_UNORM_SRGB: - case DXGI_FORMAT_BC4_TYPELESS: - case DXGI_FORMAT_BC4_UNORM: - case DXGI_FORMAT_BC4_SNORM: - case DXGI_FORMAT_BC5_TYPELESS: - case DXGI_FORMAT_BC5_UNORM: - case DXGI_FORMAT_BC5_SNORM: - // Less than 1 byte per pixel - needs special consideration - bytesPerPixel = 0; - break; - - default: - bytesPerPixel = 0; - _ASSERT(FALSE); - break; - } - return bytesPerPixel; -} - -cl_image_format D3D10Object::getCLFormatFromDXGI(DXGI_FORMAT dxgiFmt) { - cl_image_format fmt; - - //! @todo [odintsov]: add real fmt conversion from DXGI to CL - fmt.image_channel_order = 0; // CL_RGBA; - fmt.image_channel_data_type = 0; // CL_UNSIGNED_INT8; - - switch (dxgiFmt) { - case DXGI_FORMAT_R32G32B32A32_TYPELESS: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R32G32B32A32_FLOAT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case DXGI_FORMAT_R32G32B32A32_UINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNSIGNED_INT32; - break; - - case DXGI_FORMAT_R32G32B32A32_SINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SIGNED_INT32; - break; - - case DXGI_FORMAT_R32G32B32_TYPELESS: - fmt.image_channel_order = CL_RGB; - break; - - case DXGI_FORMAT_R32G32B32_FLOAT: - fmt.image_channel_order = CL_RGB; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case DXGI_FORMAT_R32G32B32_UINT: - fmt.image_channel_order = CL_RGB; - fmt.image_channel_data_type = CL_UNSIGNED_INT32; - break; - - case DXGI_FORMAT_R32G32B32_SINT: - fmt.image_channel_order = CL_RGB; - fmt.image_channel_data_type = CL_SIGNED_INT32; - break; - - case DXGI_FORMAT_R16G16B16A16_TYPELESS: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R16G16B16A16_FLOAT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case DXGI_FORMAT_R16G16B16A16_UNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case DXGI_FORMAT_R16G16B16A16_UINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNSIGNED_INT16; - break; - - case DXGI_FORMAT_R16G16B16A16_SNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SNORM_INT16; - break; - - case DXGI_FORMAT_R16G16B16A16_SINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SIGNED_INT16; - break; - - case DXGI_FORMAT_R32G32_TYPELESS: - fmt.image_channel_order = CL_RG; - break; - - case DXGI_FORMAT_R32G32_FLOAT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case DXGI_FORMAT_R32G32_UINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNSIGNED_INT32; - break; - - case DXGI_FORMAT_R32G32_SINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SIGNED_INT32; - break; - - case DXGI_FORMAT_R32G8X24_TYPELESS: - break; - - case DXGI_FORMAT_D32_FLOAT_S8X24_UINT: - break; - - case DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS: - break; - - case DXGI_FORMAT_X32_TYPELESS_G8X24_UINT: - break; - - case DXGI_FORMAT_R10G10B10A2_TYPELESS: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R10G10B10A2_UNORM: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R10G10B10A2_UINT: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R11G11B10_FLOAT: - fmt.image_channel_order = CL_RGB; - break; - - case DXGI_FORMAT_R8G8B8A8_TYPELESS: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R8G8B8A8_UNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8G8B8A8_UINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNSIGNED_INT8; - break; - - case DXGI_FORMAT_R8G8B8A8_SNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SNORM_INT8; - break; - - case DXGI_FORMAT_R8G8B8A8_SINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SIGNED_INT8; - break; - - case DXGI_FORMAT_R16G16_TYPELESS: - fmt.image_channel_order = CL_RG; - break; - - case DXGI_FORMAT_R16G16_FLOAT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case DXGI_FORMAT_R16G16_UNORM: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case DXGI_FORMAT_R16G16_UINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNSIGNED_INT16; - break; - - case DXGI_FORMAT_R16G16_SNORM: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SNORM_INT16; - break; - - case DXGI_FORMAT_R16G16_SINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SIGNED_INT16; - break; - - case DXGI_FORMAT_R32_TYPELESS: - fmt.image_channel_order = CL_R; - break; - - case DXGI_FORMAT_D32_FLOAT: - break; - - case DXGI_FORMAT_R32_FLOAT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case DXGI_FORMAT_R32_UINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNSIGNED_INT32; - break; - - case DXGI_FORMAT_R32_SINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SIGNED_INT32; - break; - - case DXGI_FORMAT_R24G8_TYPELESS: - fmt.image_channel_order = CL_RG; - break; - - case DXGI_FORMAT_D24_UNORM_S8_UINT: - break; - - case DXGI_FORMAT_R24_UNORM_X8_TYPELESS: - break; - - case DXGI_FORMAT_X24_TYPELESS_G8_UINT: - break; - - case DXGI_FORMAT_R9G9B9E5_SHAREDEXP: - break; - - case DXGI_FORMAT_R8G8_B8G8_UNORM: - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_G8R8_G8B8_UNORM: - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_B8G8R8A8_UNORM: - fmt.image_channel_order = CL_BGRA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_B8G8R8X8_UNORM: - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8G8_TYPELESS: - fmt.image_channel_order = CL_RG; - break; - - case DXGI_FORMAT_R8G8_UNORM: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8G8_UINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNSIGNED_INT8; - break; - - case DXGI_FORMAT_R8G8_SNORM: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SNORM_INT8; - break; - - case DXGI_FORMAT_R8G8_SINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SIGNED_INT8; - break; - - case DXGI_FORMAT_R16_TYPELESS: - fmt.image_channel_order = CL_R; - break; - - case DXGI_FORMAT_R16_FLOAT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case DXGI_FORMAT_D16_UNORM: - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case DXGI_FORMAT_R16_UNORM: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case DXGI_FORMAT_R16_UINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNSIGNED_INT16; - break; - - case DXGI_FORMAT_R16_SNORM: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SNORM_INT16; - break; - - case DXGI_FORMAT_R16_SINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SIGNED_INT16; - break; - - case DXGI_FORMAT_B5G6R5_UNORM: - fmt.image_channel_data_type = CL_UNORM_SHORT_565; - break; - - case DXGI_FORMAT_B5G5R5A1_UNORM: - fmt.image_channel_order = CL_BGRA; - break; - - case DXGI_FORMAT_R8_TYPELESS: - fmt.image_channel_order = CL_R; - break; - - case DXGI_FORMAT_R8_UNORM: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8_UINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNSIGNED_INT8; - break; - - case DXGI_FORMAT_R8_SNORM: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SNORM_INT8; - break; - - case DXGI_FORMAT_R8_SINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SIGNED_INT8; - break; - - case DXGI_FORMAT_A8_UNORM: - fmt.image_channel_order = CL_A; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R1_UNORM: - fmt.image_channel_order = CL_R; - break; - - case DXGI_FORMAT_BC1_TYPELESS: - case DXGI_FORMAT_BC1_UNORM: - case DXGI_FORMAT_BC1_UNORM_SRGB: - case DXGI_FORMAT_BC2_TYPELESS: - case DXGI_FORMAT_BC2_UNORM: - case DXGI_FORMAT_BC2_UNORM_SRGB: - case DXGI_FORMAT_BC3_TYPELESS: - case DXGI_FORMAT_BC3_UNORM: - case DXGI_FORMAT_BC3_UNORM_SRGB: - case DXGI_FORMAT_BC4_TYPELESS: - case DXGI_FORMAT_BC4_UNORM: - case DXGI_FORMAT_BC4_SNORM: - case DXGI_FORMAT_BC5_TYPELESS: - case DXGI_FORMAT_BC5_UNORM: - case DXGI_FORMAT_BC5_SNORM: - break; - - default: - _ASSERT(FALSE); - break; - } - - return fmt; -} - -size_t D3D10Object::getResourceByteSize() { - size_t bytes = 1; - - //! @todo [odintsov]: take into consideration the mip level?! - - switch (objDesc_.objDim_) { - case D3D10_RESOURCE_DIMENSION_BUFFER: - bytes = objDesc_.objSize_.ByteWidth; - break; - - case D3D10_RESOURCE_DIMENSION_TEXTURE3D: - bytes = objDesc_.objSize_.Depth; - - case D3D10_RESOURCE_DIMENSION_TEXTURE2D: - bytes *= objDesc_.objSize_.Height; - - case D3D10_RESOURCE_DIMENSION_TEXTURE1D: - bytes *= objDesc_.objSize_.Width * getElementBytes(); - break; - - default: - LogError("getResourceByteSize: unknown type of D3D10 resource"); - bytes = 0; - break; - } - return bytes; -} - -int D3D10Object::initD3D10Object(const Context& amdContext, ID3D10Resource* pRes, UINT subres, - D3D10Object& obj) { - ID3D10Device* pDev; - HRESULT hr; - ScopedLock sl(resLock_); - - // Check if this ressource has already been used for interop - for (const auto& it : resources_) { - if (it.first == (void*)pRes && it.second == subres) { - return CL_INVALID_D3D10_RESOURCE_KHR; - } - } - - (obj.pD3D10Res_ = pRes)->GetDevice(&pDev); - - if (!pDev) { - return CL_INVALID_D3D10_DEVICE_KHR; - } - - D3D10_QUERY_DESC desc = {D3D10_QUERY_EVENT, 0}; - pDev->CreateQuery(&desc, &obj.pQuery_); - -#define SET_SHARED_FLAGS() \ - { \ - obj.pD3D10ResOrig_ = obj.pD3D10Res_; \ - memcpy(&obj.objDescOrig_, &obj.objDesc_, sizeof(D3D10ObjDesc_t)); \ - /* @todo - Check device type and select right usage for resource */ \ - /* For now get only DPU path, CPU path for buffers */ \ - /* will not worl on DEFAUL resources */ \ - /*desc.Usage = D3D10_USAGE_STAGING;*/ \ - desc.Usage = D3D10_USAGE_DEFAULT; \ - desc.MiscFlags = D3D10_RESOURCE_MISC_SHARED; \ - desc.CPUAccessFlags = 0; \ - } - -#define STORE_SHARED_FLAGS(restype) \ - { \ - if (S_OK == hr && obj.pD3D10Res_) { \ - obj.objDesc_.objFlags_.d3d10Usage_ = desc.Usage; \ - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; \ - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; \ - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; \ - } else { \ - LogError("\nCannot create shared " #restype "\n"); \ - return CL_INVALID_D3D10_RESOURCE_KHR; \ - } \ - } - -#define SET_BINDING() \ - { \ - switch (desc.Format) { \ - case DXGI_FORMAT_D32_FLOAT_S8X24_UINT: \ - case DXGI_FORMAT_D32_FLOAT: \ - case DXGI_FORMAT_D24_UNORM_S8_UINT: \ - case DXGI_FORMAT_D16_UNORM: \ - desc.BindFlags = D3D10_BIND_DEPTH_STENCIL; \ - break; \ - default: \ - desc.BindFlags = D3D10_BIND_SHADER_RESOURCE | D3D10_BIND_RENDER_TARGET; \ - break; \ - } \ - } - - pRes->GetType(&obj.objDesc_.objDim_); - - // Init defaults - obj.objDesc_.objSize_.Height = 1; - obj.objDesc_.objSize_.Depth = 1; - obj.objDesc_.mipLevels_ = 1; - obj.objDesc_.arraySize_ = 1; - obj.objDesc_.dxgiFormat_ = DXGI_FORMAT_UNKNOWN; - obj.objDesc_.dxgiSampleDesc_ = dxgiSampleDescDefault; - - switch (obj.objDesc_.objDim_) { - case D3D10_RESOURCE_DIMENSION_BUFFER: // = 1, - { - D3D10_BUFFER_DESC desc; - (reinterpret_cast(pRes))->GetDesc(&desc); - obj.objDesc_.objSize_.ByteWidth = desc.ByteWidth; - obj.objDesc_.objFlags_.d3d10Usage_ = desc.Usage; - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; - // Handle D3D10Buffer without shared handle - create - // a duplicate with shared handle to provide for CAL - if (!(obj.objDesc_.objFlags_.miscFlags_ & D3D10_RESOURCE_MISC_SHARED)) { - SET_SHARED_FLAGS(); - desc.BindFlags = D3D10_BIND_SHADER_RESOURCE | D3D10_BIND_RENDER_TARGET; - hr = pDev->CreateBuffer(&desc, NULL, (ID3D10Buffer**)&obj.pD3D10Res_); - STORE_SHARED_FLAGS(ID3D10Buffer); - } - } break; - - case D3D10_RESOURCE_DIMENSION_TEXTURE1D: // = 2, - { - D3D10_TEXTURE1D_DESC desc; - (reinterpret_cast(pRes))->GetDesc(&desc); - - if (subres) { - // Calculate correct size of the subresource - UINT miplevel = subres; - if (desc.ArraySize > 1) { - miplevel = subres % desc.ArraySize; - } - if (miplevel >= desc.MipLevels) { - LogWarning("\nMiplevel >= number of miplevels\n"); - } - if (subres >= desc.MipLevels * desc.ArraySize) { - return CL_INVALID_VALUE; - } - desc.Width >>= miplevel; - if (!desc.Width) { - desc.Width = 1; - } - } - obj.objDesc_.objSize_.Width = desc.Width; - obj.objDesc_.mipLevels_ = desc.MipLevels; - obj.objDesc_.arraySize_ = desc.ArraySize; - obj.objDesc_.dxgiFormat_ = desc.Format; - obj.objDesc_.objFlags_.d3d10Usage_ = desc.Usage; - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; - // Handle D3D10Texture1D without shared handle - create - // a duplicate with shared handle and provide it for CAL - // Workaround for subresource > 0 in shared resource - if (subres) obj.objDesc_.objFlags_.miscFlags_ &= ~(D3D10_RESOURCE_MISC_SHARED); - if (!(obj.objDesc_.objFlags_.miscFlags_ & D3D10_RESOURCE_MISC_SHARED)) { - SET_SHARED_FLAGS(); - SET_BINDING(); - obj.objDesc_.mipLevels_ = desc.MipLevels = 1; - obj.objDesc_.arraySize_ = desc.ArraySize = 1; - hr = pDev->CreateTexture1D(&desc, NULL, (ID3D10Texture1D**)&obj.pD3D10Res_); - STORE_SHARED_FLAGS(ID3D10Texture1D); - } - } break; - - case D3D10_RESOURCE_DIMENSION_TEXTURE2D: // = 3, - { - D3D10_TEXTURE2D_DESC desc; - (reinterpret_cast(pRes))->GetDesc(&desc); - - if (subres) { - // Calculate correct size of the subresource - UINT miplevel = subres; - if (desc.ArraySize > 1) { - miplevel = subres % desc.MipLevels; - } - if (miplevel >= desc.MipLevels) { - LogWarning("\nMiplevel >= number of miplevels\n"); - } - if (subres >= desc.MipLevels * desc.ArraySize) { - return CL_INVALID_VALUE; - } - desc.Width >>= miplevel; - if (!desc.Width) { - desc.Width = 1; - } - desc.Height >>= miplevel; - if (!desc.Height) { - desc.Height = 1; - } - } - obj.objDesc_.objSize_.Width = desc.Width; - obj.objDesc_.objSize_.Height = desc.Height; - obj.objDesc_.mipLevels_ = desc.MipLevels; - obj.objDesc_.arraySize_ = desc.ArraySize; - obj.objDesc_.dxgiFormat_ = desc.Format; - obj.objDesc_.dxgiSampleDesc_ = desc.SampleDesc; - obj.objDesc_.objFlags_.d3d10Usage_ = desc.Usage; - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; - // Handle D3D10Texture2D without shared handle - create - // a duplicate with shared handle and provide it for CAL - // Workaround for subresource > 0 in shared resource - if (subres) obj.objDesc_.objFlags_.miscFlags_ &= ~(D3D10_RESOURCE_MISC_SHARED); - if (!(obj.objDesc_.objFlags_.miscFlags_ & D3D10_RESOURCE_MISC_SHARED)) { - SET_SHARED_FLAGS(); - SET_BINDING(); - obj.objDesc_.mipLevels_ = desc.MipLevels = 1; - obj.objDesc_.arraySize_ = desc.ArraySize = 1; - hr = pDev->CreateTexture2D(&desc, NULL, (ID3D10Texture2D**)&obj.pD3D10Res_); - STORE_SHARED_FLAGS(ID3D10Texture2D); - } - } break; - - case D3D10_RESOURCE_DIMENSION_TEXTURE3D: // = 4 - { - D3D10_TEXTURE3D_DESC desc; - (reinterpret_cast(pRes))->GetDesc(&desc); - - if (subres) { - // Calculate correct size of the subresource - UINT miplevel = subres; - if (miplevel >= desc.MipLevels) { - LogWarning("\nMiplevel >= number of miplevels\n"); - } - if (subres >= desc.MipLevels) { - return CL_INVALID_VALUE; - } - desc.Width >>= miplevel; - if (!desc.Width) { - desc.Width = 1; - } - desc.Height >>= miplevel; - if (!desc.Height) { - desc.Height = 1; - } - desc.Depth >>= miplevel; - if (!desc.Depth) { - desc.Depth = 1; - } - } - obj.objDesc_.objSize_.Width = desc.Width; - obj.objDesc_.objSize_.Height = desc.Height; - obj.objDesc_.objSize_.Depth = desc.Depth; - obj.objDesc_.mipLevels_ = desc.MipLevels; - obj.objDesc_.dxgiFormat_ = desc.Format; - obj.objDesc_.objFlags_.d3d10Usage_ = desc.Usage; - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; - // Handle D3D10Texture3D without shared handle - create - // a duplicate with shared handle and provide it for CAL - // Workaround for subresource > 0 in shared resource - if (obj.objDesc_.mipLevels_ > 1) - obj.objDesc_.objFlags_.miscFlags_ &= ~(D3D10_RESOURCE_MISC_SHARED); - if (!(obj.objDesc_.objFlags_.miscFlags_ & D3D10_RESOURCE_MISC_SHARED)) { - SET_SHARED_FLAGS(); - SET_BINDING(); - obj.objDesc_.mipLevels_ = desc.MipLevels = 1; - hr = pDev->CreateTexture3D(&desc, NULL, (ID3D10Texture3D**)&obj.pD3D10Res_); - STORE_SHARED_FLAGS(ID3D10Texture3D); - } - } break; - - default: - LogError("unknown type of D3D10 resource"); - return CL_INVALID_D3D10_RESOURCE_KHR; - } - obj.subRes_ = subres; - pDev->Release(); - // Check for CL format compatibilty - if (obj.objDesc_.objDim_ != D3D10_RESOURCE_DIMENSION_BUFFER) { - cl_image_format clFmt = obj.getCLFormatFromDXGI(obj.objDesc_.dxgiFormat_); - amd::Image::Format imageFormat(clFmt); - if (!imageFormat.isSupported(amdContext)) { - return CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - } - } - resources_.push_back({pRes, subres}); - return CL_SUCCESS; -} - -bool D3D10Object::copyOrigToShared() { - // Don't copy if there is no orig - if (NULL == getD3D10ResOrig()) return true; - - ID3D10Device* d3dDev; - pD3D10Res_->GetDevice(&d3dDev); - if (!d3dDev) { - LogError("\nCannot get D3D10 device from D3D10 resource\n"); - return false; - } - // Any usage source can be read by GPU - d3dDev->CopySubresourceRegion(pD3D10Res_, 0, 0, 0, 0, pD3D10ResOrig_, subRes_, NULL); - - // Flush D3D queues and make sure D3D stuff is finished - d3dDev->Flush(); - pQuery_->End(); - BOOL data = FALSE; - while ((S_OK != pQuery_->GetData(&data, sizeof(BOOL), 0)) || (data != TRUE)) { - } - - d3dDev->Release(); - return true; -} - -bool D3D10Object::copySharedToOrig() { - // Don't copy if there is no orig - if (NULL == getD3D10ResOrig()) return true; - - ID3D10Device* d3dDev; - pD3D10Res_->GetDevice(&d3dDev); - if (!d3dDev) { - LogError("\nCannot get D3D10 device from D3D10 resource\n"); - return false; - } - - d3dDev->CopySubresourceRegion(pD3D10ResOrig_, subRes_, 0, 0, 0, pD3D10Res_, 0, NULL); - - d3dDev->Release(); - return true; -} - -std::vector> D3D10Object::resources_; -Monitor D3D10Object::resLock_; - -// -// Class BufferD3D10 implementation -// -void BufferD3D10::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(BufferD3D10)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -// -// Class Image1DD3D10 implementation -// - -void Image1DD3D10::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(Image1DD3D10)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -// -// Class Image2DD3D10 implementation -// - -void Image2DD3D10::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(Image2DD3D10)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -// -// Class Image3DD3D10 implementation -// -void Image3DD3D10::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(Image3DD3D10)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -} // namespace amd - -#endif //_WIN32 diff --git a/src/hipamd/src/cl_d3d11.cpp b/src/hipamd/src/cl_d3d11.cpp deleted file mode 100644 index f82e6d50be..0000000000 --- a/src/hipamd/src/cl_d3d11.cpp +++ /dev/null @@ -1,1571 +0,0 @@ -/* Copyright (c) 2009-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifdef _WIN32 - -#include "top.hpp" - -#include "cl_d3d11_amd.hpp" -#include "platform/command.hpp" - -#include -#include - -/*! \addtogroup API - * @{ - * - * \addtogroup CL_D3D11_Interops - * - * This section discusses OpenCL functions that allow applications to use Direct3D 11 - * resources (buffers/textures) as OpenCL memory objects. This allows efficient sharing of - * data between OpenCL and Direct3D 11. The OpenCL API can be used to execute kernels that - * read and/or write memory objects that are also the Direct3D resources. - * An OpenCL image object can be created from a D3D11 texture object. An - * OpenCL buffer object can be created from a D3D11 buffer object (index/vertex). - * - * @} - * \addtogroup clGetDeviceIDsFromD3D11KHR - * @{ - */ - -RUNTIME_ENTRY(cl_int, clGetDeviceIDsFromD3D11KHR, - (cl_platform_id platform, cl_d3d11_device_source_khr d3d_device_source, - void* d3d_object, cl_d3d11_device_set_khr d3d_device_set, cl_uint num_entries, - cl_device_id* devices, cl_uint* num_devices)) { - cl_int errcode; - ID3D11Device* d3d11_device = NULL; - cl_device_id* gpu_devices; - cl_uint num_gpu_devices = 0; - bool create_d3d11Device = false; - static const bool VALIDATE_ONLY = true; - HMODULE d3d11Module = NULL; - - if (platform != NULL && platform != AMD_PLATFORM) { - LogWarning("\"platrform\" is not a valid AMD platform"); - return CL_INVALID_PLATFORM; - } - if (((num_entries > 0 || num_devices == NULL) && devices == NULL) || - (num_entries == 0 && devices != NULL)) { - return CL_INVALID_VALUE; - } - // Get GPU devices - errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 0, NULL, &num_gpu_devices); - if (errcode != CL_SUCCESS && errcode != CL_DEVICE_NOT_FOUND) { - return CL_INVALID_VALUE; - } - - if (!num_gpu_devices) { - *not_null(num_devices) = 0; - return CL_DEVICE_NOT_FOUND; - } - - switch (d3d_device_source) { - case CL_D3D11_DEVICE_KHR: - d3d11_device = static_cast(d3d_object); - break; - case CL_D3D11_DXGI_ADAPTER_KHR: { - static PFN_D3D11_CREATE_DEVICE dynamicD3D11CreateDevice = NULL; - - d3d11Module = LoadLibrary("D3D11.dll"); - if (d3d11Module == NULL) { - return CL_INVALID_PLATFORM; - } - - dynamicD3D11CreateDevice = - (PFN_D3D11_CREATE_DEVICE)GetProcAddress(d3d11Module, "D3D11CreateDevice"); - - IDXGIAdapter* dxgi_adapter = static_cast(d3d_object); - D3D_FEATURE_LEVEL requestedFeatureLevels[] = {D3D_FEATURE_LEVEL_10_0}; - D3D_FEATURE_LEVEL featureLevel = D3D_FEATURE_LEVEL_11_0; - HRESULT hr = dynamicD3D11CreateDevice(dxgi_adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, - requestedFeatureLevels, 1, D3D11_SDK_VERSION, - &d3d11_device, &featureLevel, NULL); - if (SUCCEEDED(hr) && (NULL != d3d11_device)) { - create_d3d11Device = true; - } else { - FreeLibrary(d3d11Module); - return CL_INVALID_VALUE; - } - } break; - default: - LogWarning("\"d3d_device_source\" is invalid"); - return CL_INVALID_VALUE; - } - - switch (d3d_device_set) { - case CL_PREFERRED_DEVICES_FOR_D3D11_KHR: - case CL_ALL_DEVICES_FOR_D3D11_KHR: { - gpu_devices = (cl_device_id*)alloca(num_gpu_devices * sizeof(cl_device_id)); - - errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, num_gpu_devices, gpu_devices, NULL); - if (errcode != CL_SUCCESS) { - break; - } - - std::vector compatible_devices; - for (cl_uint i = 0; i < num_gpu_devices; ++i) { - void* external_device[amd::Context::DeviceFlagIdx::LastDeviceFlagIdx] = {}; - external_device[amd::Context::DeviceFlagIdx::D3D11DeviceKhrIdx] = d3d11_device; - - cl_device_id device = gpu_devices[i]; - if (is_valid(device) && - as_amd(device)->bindExternalDevice(amd::Context::Flags::D3D11DeviceKhr, external_device, - NULL, VALIDATE_ONLY)) { - compatible_devices.push_back(as_amd(device)); - } - } - if (compatible_devices.size() == 0) { - *not_null(num_devices) = 0; - errcode = CL_DEVICE_NOT_FOUND; - break; - } - - auto it = compatible_devices.cbegin(); - cl_uint compatible_count = std::min(num_entries, (cl_uint)compatible_devices.size()); - - while (compatible_count--) { - *devices++ = as_cl(*it++); - --num_entries; - } - while (num_entries--) { - *devices++ = (cl_device_id)0; - } - - *not_null(num_devices) = (cl_uint)compatible_devices.size(); - } break; - - default: - LogWarning("\"d3d_device_set\" is invalid"); - errcode = CL_INVALID_VALUE; - } - - if (create_d3d11Device) { - d3d11_device->Release(); - FreeLibrary(d3d11Module); - } - return errcode; -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromD3D11BufferKHR - * @{ - */ - -/*! \brief Creates an OpenCL buffer object from a Direct3D 10 resource. - * - * \param context is a valid OpenCL context. - * - * \param flags is a bit-field that is used to specify usage information. - * Only CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE values - * can be used. - * - * \param pD3DResource is a valid pointer to a D3D11 resource of type ID3D11Buffer. - * - * \return valid non-zero OpenCL buffer object and \a errcode_ret is set - * to CL_SUCCESS if the buffer object is created successfully. It returns a NULL - * value with one of the following error values returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a context is not a valid context or if Direct3D 10 - * interoperatbility has not been initialized between context and the ID3D11Device - * from which pD3DResource was created. - * - CL_INVALID_VALUE if values specified in \a clFlags are not valid. - * - CL_INVALID_D3D_RESOURCE if \a pD3DResource is not of type ID3D11Buffer. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.0r33? - */ - -RUNTIME_ENTRY_RET(cl_mem, clCreateFromD3D11BufferKHR, - (cl_context context, cl_mem_flags flags, ID3D11Buffer* pD3DResource, - cl_int* errcode_ret)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - if (!flags) flags = CL_MEM_READ_WRITE; - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - if (!pD3DResource) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("parameter \"pD3DResource\" is a NULL pointer"); - return clMemObj; - } - return ( - amd::clCreateBufferFromD3D11ResourceAMD(*as_amd(context), flags, pD3DResource, errcode_ret)); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateImageFromD3D11Resource - * @{ - */ - -/*! \brief Create an OpenCL 2D or 3D image object from a D3D11 resource. - * - * \param context is a valid OpenCL context. - * - * \param flags is a bit-field that is used to specify usage information. - * Only CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE values - * can be used. - * - * \param pD3DResource is a valid pointer to a D3D11 resource of type - * ID3D11Texture2D, ID3D11Texture2D, or ID3D11Texture3D. - * If pD3DResource is of type ID3D11Texture1D then the created image object - * will be a 1D mipmapped image object. - * If pD3DResource is of type ID3D11Texture2D and was not created with flag - * D3D11_RESOURCE_MISC_TEXTURECUBE then the created image object will be a - * 2D mipmapped image object. - * If pD3DResource is of type ID3D11Texture2D and was created with flag - * D3D11_RESOURCE_MISC_TEXTURECUBE then the created image object will be - * a cubemap mipmapped image object. - * errocde_ret returns CL_INVALID_D3D_RESOURCE if an OpenCL memory object has - * already been created from pD3DResource in context. - * If pD3DResource is of type ID3D11Texture3D then the created image object will - * be a 3D mipmapped imageobject. - * - * \return valid non-zero OpenCL image object and \a errcode_ret is set - * to CL_SUCCESS if the image object is created successfully. It returns a NULL - * value with one of the following error values returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a context is not a valid context or if Direct3D 11 - * interoperatbility has not been initialized between context and the ID3D11Device - * from which pD3DResource was created. - * - CL_INVALID_VALUE if values specified in \a flags are not valid. - * - CL_INVALID_D3D_RESOURCE if \a pD3DResource is not of type ID3D11Texture1D, - * ID3D11Texture2D, or ID3D11Texture3D. - * - CL_INVALID_D3D_RESOURCE if an OpenCL memory object has already been created - * from \a pD3DResource in context. - * - CL_INVALID_IMAGE_FORMAT if the Direct3D 11 texture format does not map - * to an appropriate OpenCL image format. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.0r48? - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateImageFromD3D11Resource, - (cl_context context, cl_mem_flags flags, ID3D11Resource* pD3DResource, - UINT subresource, int* errcode_ret, UINT dimension)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - if (!flags) flags = CL_MEM_READ_WRITE; - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - if (!pD3DResource) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("parameter \"pD3DResource\" is a NULL pointer"); - return clMemObj; - } - - // Verify context init'ed for interop - ID3D11Device* pDev; - pD3DResource->GetDevice(&pDev); - if (pDev == NULL) { - *not_null(errcode_ret) = CL_INVALID_D3D11_DEVICE_KHR; - LogWarning("Cannot retrieve D3D11 device from D3D11 resource"); - return (cl_mem)0; - } - pDev->Release(); - if (!((*as_amd(context)).info().flags_ & amd::Context::D3D11DeviceKhr)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("\"amdContext\" is not created from D3D11 device"); - return (cl_mem)0; - } - - // Check for image support - const std::vector& devices = as_amd(context)->devices(); - bool supportPass = false; - bool sizePass = false; - for (const auto& it : devices) { - if (it->info().imageSupport_) { - supportPass = true; - } - } - if (!supportPass) { - *not_null(errcode_ret) = CL_INVALID_OPERATION; - LogWarning("there are no devices in context to support images"); - return (cl_mem)0; - } - - switch (dimension) { -#if 0 - case 1: - return(amd::clCreateImage1DFromD3D11ResourceAMD( - *as_amd(context), - flags, - pD3DResource, - subresource, - errcode_ret)); -#endif // 0 - case 2: - return (amd::clCreateImage2DFromD3D11ResourceAMD(*as_amd(context), flags, pD3DResource, - subresource, errcode_ret)); - case 3: - return (amd::clCreateImage3DFromD3D11ResourceAMD(*as_amd(context), flags, pD3DResource, - subresource, errcode_ret)); - default: - break; - } - - *not_null(errcode_ret) = CL_INVALID_D3D11_RESOURCE_KHR; - return (cl_mem)0; -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromD3D11Texture2DKHR - * @{ - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromD3D11Texture2DKHR, - (cl_context context, cl_mem_flags flags, ID3D11Texture2D* resource, - UINT subresource, cl_int* errcode_ret)) { - return clCreateImageFromD3D11Resource(context, flags, resource, subresource, errcode_ret, 2); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromD3D11Texture3DKHR - * @{ - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromD3D11Texture3DKHR, - (cl_context context, cl_mem_flags flags, ID3D11Texture3D* resource, - UINT subresource, cl_int* errcode_ret)) { - return clCreateImageFromD3D11Resource(context, flags, resource, subresource, errcode_ret, 3); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clEnqueueAcquireD3D11ObjectsKHR - * @{ - */ -RUNTIME_ENTRY(cl_int, clEnqueueAcquireD3D11ObjectsKHR, - (cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return amd::clEnqueueAcquireExtObjectsAMD(command_queue, num_objects, mem_objects, - num_events_in_wait_list, event_wait_list, event, - CL_COMMAND_ACQUIRE_D3D11_OBJECTS_KHR); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clEnqueueReleaseD3D11ObjectsKHR - * @{ - */ -RUNTIME_ENTRY(cl_int, clEnqueueReleaseD3D11ObjectsKHR, - (cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return amd::clEnqueueReleaseExtObjectsAMD(command_queue, num_objects, mem_objects, - num_events_in_wait_list, event_wait_list, event, - CL_COMMAND_RELEASE_D3D11_OBJECTS_KHR); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clGetPlaneFromImageAMD - * @{ - */ -RUNTIME_ENTRY_RET(cl_mem, clGetPlaneFromImageAMD, - (cl_context context, cl_mem mem, cl_uint plane, cl_int* errcode_ret)) { - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return 0; - } - if (mem == 0) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - return 0; - } - if (!is_valid(mem)) { - *not_null(errcode_ret) = CL_INVALID_MEM_OBJECT; - return 0; - } - amd::Memory* amdMem = as_amd(mem); - amd::Context& amdContext = *as_amd(context); - if (amdMem->getInteropObj() == NULL) { - *not_null(errcode_ret) = CL_INVALID_MEM_OBJECT; - return 0; - } - amd::Image2DD3D11* pImage = reinterpret_cast(amdMem); - ID3D11Resource* pD3DResource = pImage->getD3D11Resource(); - // Verify the resource is a 2D texture - D3D11_RESOURCE_DIMENSION rType; - pD3DResource->GetType(&rType); - if (rType != D3D11_RESOURCE_DIMENSION_TEXTURE2D) { - *not_null(errcode_ret) = CL_INVALID_D3D11_RESOURCE_KHR; - return (cl_mem)0; - } - - amd::D3D11Object obj; - int errcode = amd::D3D11Object::initD3D11Object(amdContext, pD3DResource, 0, obj, plane); - if (CL_SUCCESS != errcode) { - *not_null(errcode_ret) = errcode; - return (cl_mem)0; - } - - amd::Image2DD3D11* pImage2DD3D11 = - new (amdContext) amd::Image2DD3D11(amdContext, pImage->getMemFlags(), obj); - if (!pImage2DD3D11) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem)0; - } - if (!pImage2DD3D11->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImage2DD3D11->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImage2DD3D11); -} -RUNTIME_EXIT - -// -// -// namespace amd -// -// -namespace amd { -/*! @} - * \addtogroup CL-D3D11 interop helper functions - * @{ - */ - - -//******************************************************************* -// -// Internal implementation of CL API functions -// -//******************************************************************* -// -// clCreateBufferFromD3D11ResourceAMD -// -cl_mem clCreateBufferFromD3D11ResourceAMD(Context& amdContext, cl_mem_flags flags, - ID3D11Resource* pD3DResource, int* errcode_ret) { - // Verify pD3DResource is a buffer - D3D11_RESOURCE_DIMENSION rType; - pD3DResource->GetType(&rType); - if (rType != D3D11_RESOURCE_DIMENSION_BUFFER) { - *not_null(errcode_ret) = CL_INVALID_D3D11_RESOURCE_KHR; - return (cl_mem)0; - } - - D3D11Object obj; - int errcode = D3D11Object::initD3D11Object(amdContext, pD3DResource, 0, obj); - if (CL_SUCCESS != errcode) { - *not_null(errcode_ret) = errcode; - return (cl_mem)0; - } - - BufferD3D11* pBufferD3D11 = new (amdContext) BufferD3D11(amdContext, flags, obj); - if (!pBufferD3D11) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem)0; - } - if (!pBufferD3D11->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pBufferD3D11->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pBufferD3D11); -} - -// -// clCreateImage2DFromD3D11ResourceAMD -// -cl_mem clCreateImage2DFromD3D11ResourceAMD(Context& amdContext, cl_mem_flags flags, - ID3D11Resource* pD3DResource, UINT subresource, - int* errcode_ret) { - // Verify the resource is a 2D texture - D3D11_RESOURCE_DIMENSION rType; - pD3DResource->GetType(&rType); - if (rType != D3D11_RESOURCE_DIMENSION_TEXTURE2D) { - *not_null(errcode_ret) = CL_INVALID_D3D11_RESOURCE_KHR; - return (cl_mem)0; - } - - D3D11Object obj; - int errcode = D3D11Object::initD3D11Object(amdContext, pD3DResource, subresource, obj); - if (CL_SUCCESS != errcode) { - *not_null(errcode_ret) = errcode; - return (cl_mem)0; - } - - Image2DD3D11* pImage2DD3D11 = new (amdContext) Image2DD3D11(amdContext, flags, obj); - if (!pImage2DD3D11) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem)0; - } - if (!pImage2DD3D11->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImage2DD3D11->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImage2DD3D11); -} - -// -// clCreateImage2DFromD3D11ResourceAMD -// -cl_mem clCreateImage3DFromD3D11ResourceAMD(Context& amdContext, cl_mem_flags flags, - ID3D11Resource* pD3DResource, UINT subresource, - int* errcode_ret) { - // Verify the resource is a 2D texture - D3D11_RESOURCE_DIMENSION rType; - pD3DResource->GetType(&rType); - if (rType != D3D11_RESOURCE_DIMENSION_TEXTURE3D) { - *not_null(errcode_ret) = CL_INVALID_D3D11_RESOURCE_KHR; - return (cl_mem)0; - } - - D3D11Object obj; - int errcode = D3D11Object::initD3D11Object(amdContext, pD3DResource, subresource, obj); - if (CL_SUCCESS != errcode) { - *not_null(errcode_ret) = errcode; - return (cl_mem)0; - } - - Image3DD3D11* pImage3DD3D11 = new (amdContext) Image3DD3D11(amdContext, flags, obj); - if (!pImage3DD3D11) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem)0; - } - if (!pImage3DD3D11->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImage3DD3D11->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImage3DD3D11); -} - -size_t D3D11Object::getResourceByteSize() { - size_t bytes = 1; - - //! @todo [odintsov]: take into consideration the mip level?! - - switch (objDesc_.objDim_) { - case D3D11_RESOURCE_DIMENSION_BUFFER: - bytes = objDesc_.objSize_.ByteWidth; - break; - - case D3D11_RESOURCE_DIMENSION_TEXTURE3D: - bytes = objDesc_.objSize_.Depth; - - case D3D11_RESOURCE_DIMENSION_TEXTURE2D: - bytes *= objDesc_.objSize_.Height; - - case D3D11_RESOURCE_DIMENSION_TEXTURE1D: - bytes *= objDesc_.objSize_.Width * getElementBytes(); - break; - - default: - LogError("getResourceByteSize: unknown type of D3D11 resource"); - bytes = 0; - break; - } - return bytes; -} - -cl_uint D3D11Object::getMiscFlag() { - if ((objDesc_.dxgiFormat_ == DXGI_FORMAT_NV12) || - (objDesc_.dxgiFormat_ == DXGI_FORMAT_P010)) { - return 1; - } - else if (objDesc_.dxgiFormat_ == DXGI_FORMAT_YUY2) { - return 3; - } - return 0; -} - -int D3D11Object::initD3D11Object(const Context& amdContext, ID3D11Resource* pRes, UINT subres, - D3D11Object& obj, INT plane) { - ID3D11Device* pDev; - HRESULT hr; - ScopedLock sl(resLock_); - - // Check if this ressource has already been used for interop - for (const auto& it : resources_) { - if (it.first == (void*)pRes && it.second.first == subres && - it.second.second == plane) { - return CL_INVALID_D3D11_RESOURCE_KHR; - } - } - - (obj.pD3D11Res_ = pRes)->GetDevice(&pDev); - - if (!pDev) { - return CL_INVALID_D3D11_DEVICE_KHR; - } - - D3D11_QUERY_DESC desc = {D3D11_QUERY_EVENT, 0}; - pDev->CreateQuery(&desc, &obj.pQuery_); - -#define SET_SHARED_FLAGS() \ - { \ - obj.pD3D11ResOrig_ = obj.pD3D11Res_; \ - /* @todo - Check device type and select right usage for resource */ \ - /* For now get only DPU path, CPU path for buffers */ \ - /* will not worl on DEFAUL resources */ \ - /*desc.Usage = D3D11_USAGE_STAGING;*/ \ - desc.Usage = D3D11_USAGE_DEFAULT; \ - desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; \ - desc.CPUAccessFlags = 0; \ - } - -#define STORE_SHARED_FLAGS_BUFFER(restype) \ - { \ - if (S_OK == hr && obj.pD3D11Res_) { \ - obj.objDesc_.objFlags_.d3d11Usage_ = desc.Usage; \ - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; \ - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; \ - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; \ - obj.objDesc_.objFlags_.structureByteStride_ = desc.StructureByteStride; \ - } else { \ - LogError("\nCannot create shared " #restype "\n"); \ - return CL_INVALID_D3D11_RESOURCE_KHR; \ - } \ - } - -#define STORE_SHARED_FLAGS(restype) \ - { \ - if (S_OK == hr && obj.pD3D11Res_) { \ - obj.objDesc_.objFlags_.d3d11Usage_ = desc.Usage; \ - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; \ - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; \ - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; \ - } else { \ - LogError("\nCannot create shared " #restype "\n"); \ - return CL_INVALID_D3D11_RESOURCE_KHR; \ - } \ - } - -#define SET_BINDING() \ - { \ - switch (desc.Format) { \ - case DXGI_FORMAT_D32_FLOAT_S8X24_UINT: \ - case DXGI_FORMAT_D32_FLOAT: \ - case DXGI_FORMAT_D24_UNORM_S8_UINT: \ - case DXGI_FORMAT_D16_UNORM: \ - desc.BindFlags = D3D11_BIND_DEPTH_STENCIL; \ - break; \ - default: \ - desc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET; \ - break; \ - } \ - } - - pRes->GetType(&obj.objDesc_.objDim_); - - // Init defaults - obj.objDesc_.objSize_.Height = 1; - obj.objDesc_.objSize_.Depth = 1; - obj.objDesc_.mipLevels_ = 1; - obj.objDesc_.arraySize_ = 1; - obj.objDesc_.dxgiFormat_ = DXGI_FORMAT_UNKNOWN; - obj.objDesc_.dxgiSampleDesc_ = dxgiSampleDescDefault; - - switch (obj.objDesc_.objDim_) { - case D3D11_RESOURCE_DIMENSION_BUFFER: // = 1, - { - D3D11_BUFFER_DESC desc; - (reinterpret_cast(pRes))->GetDesc(&desc); - obj.objDesc_.objSize_.ByteWidth = desc.ByteWidth; - obj.objDesc_.objFlags_.d3d11Usage_ = desc.Usage; - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; - obj.objDesc_.objFlags_.structureByteStride_ = desc.StructureByteStride; - // Handle D3D11Buffer without shared handle - create - // a duplicate with shared handle to provide for CAL - if (!(obj.objDesc_.objFlags_.miscFlags_ & D3D11_RESOURCE_MISC_SHARED)) { - SET_SHARED_FLAGS(); - desc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET; - hr = pDev->CreateBuffer(&desc, NULL, (ID3D11Buffer**)&obj.pD3D11Res_); - STORE_SHARED_FLAGS_BUFFER(ID3D11Buffer); - } - } break; - - case D3D11_RESOURCE_DIMENSION_TEXTURE1D: // = 2, - { - D3D11_TEXTURE1D_DESC desc; - (reinterpret_cast(pRes))->GetDesc(&desc); - - if (subres) { - // Calculate correct size of the subresource - UINT miplevel = subres; - if (desc.ArraySize > 1) { - miplevel = subres % desc.ArraySize; - } - if (miplevel >= desc.MipLevels) { - LogWarning("\nMiplevel >= number of miplevels\n"); - } - if (subres >= desc.MipLevels * desc.ArraySize) { - return CL_INVALID_VALUE; - } - desc.Width >>= miplevel; - if (!desc.Width) { - desc.Width = 1; - } - } - obj.objDesc_.objSize_.Width = desc.Width; - obj.objDesc_.mipLevels_ = desc.MipLevels; - obj.objDesc_.arraySize_ = desc.ArraySize; - obj.objDesc_.dxgiFormat_ = desc.Format; - obj.objDesc_.objFlags_.d3d11Usage_ = desc.Usage; - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; - // Handle D3D11Texture1D without shared handle - create - // a duplicate with shared handle and provide it for CAL - // Workaround for subresource > 0 in shared resource - if (subres) obj.objDesc_.objFlags_.miscFlags_ &= ~(D3D11_RESOURCE_MISC_SHARED); - if (!(obj.objDesc_.objFlags_.miscFlags_ & D3D11_RESOURCE_MISC_SHARED)) { - SET_SHARED_FLAGS(); - SET_BINDING(); - obj.objDesc_.mipLevels_ = desc.MipLevels = 1; - obj.objDesc_.arraySize_ = desc.ArraySize = 1; - hr = pDev->CreateTexture1D(&desc, NULL, (ID3D11Texture1D**)&obj.pD3D11Res_); - STORE_SHARED_FLAGS(ID3D11Texture1D); - } - } break; - - case D3D11_RESOURCE_DIMENSION_TEXTURE2D: // = 3, - { - D3D11_TEXTURE2D_DESC desc; - (reinterpret_cast(pRes))->GetDesc(&desc); - - if (subres) { - // Calculate correct size of the subresource - UINT miplevel = subres; - if (desc.ArraySize > 1) { - miplevel = subres % desc.MipLevels; - } - if (miplevel >= desc.MipLevels) { - LogWarning("\nMiplevel >= number of miplevels\n"); - } - if (subres >= desc.MipLevels * desc.ArraySize) { - return CL_INVALID_VALUE; - } - desc.Width >>= miplevel; - if (!desc.Width) { - desc.Width = 1; - } - desc.Height >>= miplevel; - if (!desc.Height) { - desc.Height = 1; - } - } - obj.objDesc_.objSize_.Width = desc.Width; - obj.objDesc_.objSize_.Height = desc.Height; - obj.objDesc_.mipLevels_ = desc.MipLevels; - obj.objDesc_.arraySize_ = desc.ArraySize; - obj.objDesc_.dxgiFormat_ = desc.Format; - obj.objDesc_.dxgiSampleDesc_ = desc.SampleDesc; - obj.objDesc_.objFlags_.d3d11Usage_ = desc.Usage; - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; - - // Handle D3D11Texture2D without shared handle - create - // a duplicate with shared handle and provide it for CAL - // Workaround for subresource > 0 in shared resource - if (subres) obj.objDesc_.objFlags_.miscFlags_ &= ~(D3D11_RESOURCE_MISC_SHARED); - if (!(obj.objDesc_.objFlags_.miscFlags_ & D3D11_RESOURCE_MISC_SHARED)) { - SET_SHARED_FLAGS(); - SET_BINDING(); - obj.objDesc_.mipLevels_ = desc.MipLevels = 1; - obj.objDesc_.arraySize_ = desc.ArraySize = 1; - hr = pDev->CreateTexture2D(&desc, NULL, (ID3D11Texture2D**)&obj.pD3D11Res_); - STORE_SHARED_FLAGS(ID3D11Texture2D); - } - - if ((desc.Format == DXGI_FORMAT_NV12) || (desc.Format == DXGI_FORMAT_P010)) { - if (plane == -1) { - obj.objDesc_.objSize_.Height += obj.objDesc_.objSize_.Height / 2; - } - if (plane == 1) { - obj.objDesc_.objSize_.Width /= 2; - obj.objDesc_.objSize_.Height /= 2; - } - } - // RGBA8 covers 2 pixels, thus divide width by 2 - if (desc.Format == DXGI_FORMAT_YUY2) { - obj.objDesc_.objSize_.Width /= 2; - } - } break; - - case D3D11_RESOURCE_DIMENSION_TEXTURE3D: // = 4 - { - D3D11_TEXTURE3D_DESC desc; - (reinterpret_cast(pRes))->GetDesc(&desc); - - if (subres) { - // Calculate correct size of the subresource - UINT miplevel = subres; - if (miplevel >= desc.MipLevels) { - LogWarning("\nMiplevel >= number of miplevels\n"); - } - if (subres >= desc.MipLevels) { - return CL_INVALID_VALUE; - } - desc.Width >>= miplevel; - if (!desc.Width) { - desc.Width = 1; - } - desc.Height >>= miplevel; - if (!desc.Height) { - desc.Height = 1; - } - desc.Depth >>= miplevel; - if (!desc.Depth) { - desc.Depth = 1; - } - } - obj.objDesc_.objSize_.Width = desc.Width; - obj.objDesc_.objSize_.Height = desc.Height; - obj.objDesc_.objSize_.Depth = desc.Depth; - obj.objDesc_.mipLevels_ = desc.MipLevels; - obj.objDesc_.dxgiFormat_ = desc.Format; - obj.objDesc_.objFlags_.d3d11Usage_ = desc.Usage; - obj.objDesc_.objFlags_.bindFlags_ = desc.BindFlags; - obj.objDesc_.objFlags_.cpuAccessFlags_ = desc.CPUAccessFlags; - obj.objDesc_.objFlags_.miscFlags_ = desc.MiscFlags; - // Handle D3D11Texture3D without shared handle - create - // a duplicate with shared handle and provide it for CAL - // Workaround for subresource > 0 in shared resource - if (obj.objDesc_.mipLevels_ > 1) - obj.objDesc_.objFlags_.miscFlags_ &= ~(D3D11_RESOURCE_MISC_SHARED); - if (!(obj.objDesc_.objFlags_.miscFlags_ & D3D11_RESOURCE_MISC_SHARED)) { - SET_SHARED_FLAGS(); - SET_BINDING(); - obj.objDesc_.mipLevels_ = desc.MipLevels = 1; - hr = pDev->CreateTexture3D(&desc, NULL, (ID3D11Texture3D**)&obj.pD3D11Res_); - STORE_SHARED_FLAGS(ID3D11Texture3D); - } - } break; - - default: - LogError("unknown type of D3D11 resource"); - return CL_INVALID_D3D11_RESOURCE_KHR; - } - obj.subRes_ = subres; - obj.plane_ = plane; - pDev->Release(); - // Check for CL format compatibilty - if (obj.objDesc_.objDim_ != D3D11_RESOURCE_DIMENSION_BUFFER) { - cl_image_format clFmt = obj.getCLFormatFromDXGI(obj.objDesc_.dxgiFormat_, plane); - amd::Image::Format imageFormat(clFmt); - if (!imageFormat.isSupported(amdContext)) { - return CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - } - } - resources_.push_back({pRes, {subres, plane}}); - return CL_SUCCESS; -} - -bool D3D11Object::copyOrigToShared() { - // Don't copy if there is no orig - if (NULL == getD3D11ResOrig()) return true; - - ID3D11Device* d3dDev; - pD3D11Res_->GetDevice(&d3dDev); - if (!d3dDev) { - LogError("\nCannot get D3D11 device from D3D11 resource\n"); - return false; - } - ID3D11DeviceContext* pImmediateContext = NULL; - d3dDev->GetImmediateContext(&pImmediateContext); - if (!pImmediateContext) { - LogError("\nCannot get D3D11 device context"); - return false; - } - assert(pD3D11ResOrig_ != NULL); - // Any usage source can be read by GPU - pImmediateContext->CopySubresourceRegion(pD3D11Res_, 0, 0, 0, 0, pD3D11ResOrig_, subRes_, NULL); - - // Flush D3D queues and make sure D3D stuff is finished - { - ScopedLock sl(resLock_); // protect from multiple - pImmediateContext->Flush(); - pImmediateContext->End(pQuery_); - BOOL data = FALSE; - while (S_OK != pImmediateContext->GetData(pQuery_, &data, sizeof(BOOL), 0)) { - } - } - - pImmediateContext->Release(); - d3dDev->Release(); - return true; -} - -bool D3D11Object::copySharedToOrig() { - // Don't copy if there is no orig - if (NULL == getD3D11ResOrig()) return true; - - ID3D11Device* d3dDev; - pD3D11Res_->GetDevice(&d3dDev); - if (!d3dDev) { - LogError("\nCannot get D3D11 device from D3D11 resource\n"); - return false; - } - ID3D11DeviceContext* pImmediateContext = NULL; - d3dDev->GetImmediateContext(&pImmediateContext); - if (!pImmediateContext) { - LogError("\nCannot get D3D11 device context"); - return false; - } - assert(pD3D11ResOrig_); - pImmediateContext->CopySubresourceRegion(pD3D11ResOrig_, subRes_, 0, 0, 0, pD3D11Res_, 0, NULL); - pImmediateContext->Release(); - - d3dDev->Release(); - return true; -} - -std::vector>> D3D11Object::resources_; -Monitor D3D11Object::resLock_; - -// -// Class BufferD3D11 implementation -// -void BufferD3D11::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(BufferD3D11)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -// -// Class Image1DD3D11 implementation -// -void Image1DD3D11::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(Image1DD3D11)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -// -// Class Image2DD3D11 implementation -// - -void Image2DD3D11::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(Image2DD3D11)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -// -// Class Image3DD3D11 implementation -// -void Image3DD3D11::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(Image3DD3D11)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -// -// Helper function SyncD3D11Objects -// -void SyncD3D11Objects(std::vector& memObjects) { - Memory*& mem = memObjects.front(); - if (!mem) { - LogWarning("\nNULL memory object\n"); - return; - } - InteropObject* interop = mem->getInteropObj(); - if (!interop) { - LogWarning("\nNULL interop object\n"); - return; - } - D3D11Object* d3dObj = interop->asD3D11Object(); - if (!d3dObj) { - LogWarning("\nNULL D3D11 object\n"); - return; - } - ID3D11Query* query = d3dObj->getQuery(); - if (!query) { - LogWarning("\nNULL ID3D11Query\n"); - return; - } - ID3D11Device* d3dDev; - query->GetDevice(&d3dDev); - if (!d3dDev) { - LogError("\nCannot get D3D11 device from D3D11 resource\n"); - return; - } - ID3D11DeviceContext* pImmediateContext = NULL; - d3dDev->GetImmediateContext(&pImmediateContext); - if (!pImmediateContext) { - LogError("\nCannot get D3D11 device context"); - return; - } - pImmediateContext->Release(); - - // Flush D3D queues and make sure D3D stuff is finished - { - ScopedLock sl(d3dObj->getResLock()); - pImmediateContext->End(query); - BOOL data = FALSE; - while ((S_OK != pImmediateContext->GetData(query, &data, sizeof(BOOL), 0)) || (data != TRUE)) { - } - } - - d3dDev->Release(); -} - -// -// Class D3D11Object implementation -// -size_t D3D11Object::getElementBytes(DXGI_FORMAT dxgiFmt, cl_uint plane) { - size_t bytesPerPixel; - - switch (dxgiFmt) { - case DXGI_FORMAT_R32G32B32A32_TYPELESS: - case DXGI_FORMAT_R32G32B32A32_FLOAT: - case DXGI_FORMAT_R32G32B32A32_UINT: - case DXGI_FORMAT_R32G32B32A32_SINT: - bytesPerPixel = 16; - break; - - case DXGI_FORMAT_R32G32B32_TYPELESS: - case DXGI_FORMAT_R32G32B32_FLOAT: - case DXGI_FORMAT_R32G32B32_UINT: - case DXGI_FORMAT_R32G32B32_SINT: - bytesPerPixel = 12; - break; - - case DXGI_FORMAT_R16G16B16A16_TYPELESS: - case DXGI_FORMAT_R16G16B16A16_FLOAT: - case DXGI_FORMAT_R16G16B16A16_UNORM: - case DXGI_FORMAT_R16G16B16A16_UINT: - case DXGI_FORMAT_R16G16B16A16_SNORM: - case DXGI_FORMAT_R16G16B16A16_SINT: - case DXGI_FORMAT_R32G32_TYPELESS: - case DXGI_FORMAT_R32G32_FLOAT: - case DXGI_FORMAT_R32G32_UINT: - case DXGI_FORMAT_R32G32_SINT: - case DXGI_FORMAT_R32G8X24_TYPELESS: - case DXGI_FORMAT_D32_FLOAT_S8X24_UINT: - case DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS: - case DXGI_FORMAT_X32_TYPELESS_G8X24_UINT: - bytesPerPixel = 8; - break; - - case DXGI_FORMAT_R10G10B10A2_TYPELESS: - case DXGI_FORMAT_R10G10B10A2_UNORM: - case DXGI_FORMAT_R10G10B10A2_UINT: - case DXGI_FORMAT_R11G11B10_FLOAT: - case DXGI_FORMAT_R8G8B8A8_TYPELESS: - case DXGI_FORMAT_R8G8B8A8_UNORM: - case DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: - case DXGI_FORMAT_R8G8B8A8_UINT: - case DXGI_FORMAT_R8G8B8A8_SNORM: - case DXGI_FORMAT_R8G8B8A8_SINT: - case DXGI_FORMAT_R16G16_TYPELESS: - case DXGI_FORMAT_R16G16_FLOAT: - case DXGI_FORMAT_R16G16_UNORM: - case DXGI_FORMAT_R16G16_UINT: - case DXGI_FORMAT_R16G16_SNORM: - case DXGI_FORMAT_R16G16_SINT: - case DXGI_FORMAT_R32_TYPELESS: - case DXGI_FORMAT_D32_FLOAT: - case DXGI_FORMAT_R32_FLOAT: - case DXGI_FORMAT_R32_UINT: - case DXGI_FORMAT_R32_SINT: - case DXGI_FORMAT_R24G8_TYPELESS: - case DXGI_FORMAT_D24_UNORM_S8_UINT: - case DXGI_FORMAT_R24_UNORM_X8_TYPELESS: - case DXGI_FORMAT_X24_TYPELESS_G8_UINT: - - case DXGI_FORMAT_R9G9B9E5_SHAREDEXP: - case DXGI_FORMAT_R8G8_B8G8_UNORM: - case DXGI_FORMAT_G8R8_G8B8_UNORM: - - case DXGI_FORMAT_B8G8R8A8_UNORM: - case DXGI_FORMAT_B8G8R8X8_UNORM: - - case DXGI_FORMAT_YUY2: - bytesPerPixel = 4; - break; - - case DXGI_FORMAT_R8G8_TYPELESS: - case DXGI_FORMAT_R8G8_UNORM: - case DXGI_FORMAT_R8G8_UINT: - case DXGI_FORMAT_R8G8_SNORM: - case DXGI_FORMAT_R8G8_SINT: - case DXGI_FORMAT_R16_TYPELESS: - case DXGI_FORMAT_R16_FLOAT: - case DXGI_FORMAT_D16_UNORM: - case DXGI_FORMAT_R16_UNORM: - case DXGI_FORMAT_R16_UINT: - case DXGI_FORMAT_R16_SNORM: - case DXGI_FORMAT_R16_SINT: - - case DXGI_FORMAT_B5G6R5_UNORM: - case DXGI_FORMAT_B5G5R5A1_UNORM: - bytesPerPixel = 2; - break; - - case DXGI_FORMAT_R8_TYPELESS: - case DXGI_FORMAT_R8_UNORM: - case DXGI_FORMAT_R8_UINT: - case DXGI_FORMAT_R8_SNORM: - case DXGI_FORMAT_R8_SINT: - case DXGI_FORMAT_A8_UNORM: - case DXGI_FORMAT_R1_UNORM: - bytesPerPixel = 1; - break; - - - case DXGI_FORMAT_BC1_TYPELESS: - case DXGI_FORMAT_BC1_UNORM: - case DXGI_FORMAT_BC1_UNORM_SRGB: - case DXGI_FORMAT_BC2_TYPELESS: - case DXGI_FORMAT_BC2_UNORM: - case DXGI_FORMAT_BC2_UNORM_SRGB: - case DXGI_FORMAT_BC3_TYPELESS: - case DXGI_FORMAT_BC3_UNORM: - case DXGI_FORMAT_BC3_UNORM_SRGB: - case DXGI_FORMAT_BC4_TYPELESS: - case DXGI_FORMAT_BC4_UNORM: - case DXGI_FORMAT_BC4_SNORM: - case DXGI_FORMAT_BC5_TYPELESS: - case DXGI_FORMAT_BC5_UNORM: - case DXGI_FORMAT_BC5_SNORM: - // Less than 1 byte per pixel - needs special consideration - bytesPerPixel = 0; - break; - case DXGI_FORMAT_NV12: - bytesPerPixel = 1; - if (plane == 1) { - bytesPerPixel = 2; - } - break; - case DXGI_FORMAT_P010: - bytesPerPixel = 2; - if (plane == 1) { - bytesPerPixel = 4; - } - break; - default: - bytesPerPixel = 0; - _ASSERT(FALSE); - break; - } - return bytesPerPixel; -} - -cl_image_format D3D11Object::getCLFormatFromDXGI(DXGI_FORMAT dxgiFmt, cl_uint plane) { - cl_image_format fmt; - - //! @todo [odintsov]: add real fmt conversion from DXGI to CL - fmt.image_channel_order = 0; // CL_RGBA; - fmt.image_channel_data_type = 0; // CL_UNSIGNED_INT8; - - switch (dxgiFmt) { - case DXGI_FORMAT_R32G32B32A32_TYPELESS: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R32G32B32A32_FLOAT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case DXGI_FORMAT_R32G32B32A32_UINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNSIGNED_INT32; - break; - - case DXGI_FORMAT_R32G32B32A32_SINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SIGNED_INT32; - break; - - case DXGI_FORMAT_R32G32B32_TYPELESS: - fmt.image_channel_order = CL_RGB; - break; - - case DXGI_FORMAT_R32G32B32_FLOAT: - fmt.image_channel_order = CL_RGB; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case DXGI_FORMAT_R32G32B32_UINT: - fmt.image_channel_order = CL_RGB; - fmt.image_channel_data_type = CL_UNSIGNED_INT32; - break; - - case DXGI_FORMAT_R32G32B32_SINT: - fmt.image_channel_order = CL_RGB; - fmt.image_channel_data_type = CL_SIGNED_INT32; - break; - - case DXGI_FORMAT_R16G16B16A16_TYPELESS: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R16G16B16A16_FLOAT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case DXGI_FORMAT_R16G16B16A16_UNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case DXGI_FORMAT_R16G16B16A16_UINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNSIGNED_INT16; - break; - - case DXGI_FORMAT_R16G16B16A16_SNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SNORM_INT16; - break; - - case DXGI_FORMAT_R16G16B16A16_SINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SIGNED_INT16; - break; - - case DXGI_FORMAT_R32G32_TYPELESS: - fmt.image_channel_order = CL_RG; - break; - - case DXGI_FORMAT_R32G32_FLOAT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case DXGI_FORMAT_R32G32_UINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNSIGNED_INT32; - break; - - case DXGI_FORMAT_R32G32_SINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SIGNED_INT32; - break; - - case DXGI_FORMAT_R32G8X24_TYPELESS: - break; - - case DXGI_FORMAT_D32_FLOAT_S8X24_UINT: - break; - - case DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS: - break; - - case DXGI_FORMAT_X32_TYPELESS_G8X24_UINT: - break; - - case DXGI_FORMAT_R10G10B10A2_TYPELESS: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R10G10B10A2_UNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT_101010; - break; - - case DXGI_FORMAT_R10G10B10A2_UINT: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R11G11B10_FLOAT: - fmt.image_channel_order = CL_RGB; - break; - - case DXGI_FORMAT_R8G8B8A8_TYPELESS: - fmt.image_channel_order = CL_RGBA; - break; - - case DXGI_FORMAT_R8G8B8A8_UNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8G8B8A8_UINT: - case DXGI_FORMAT_YUY2: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNSIGNED_INT8; - break; - - case DXGI_FORMAT_R8G8B8A8_SNORM: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SNORM_INT8; - break; - - case DXGI_FORMAT_R8G8B8A8_SINT: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_SIGNED_INT8; - break; - - case DXGI_FORMAT_R16G16_TYPELESS: - fmt.image_channel_order = CL_RG; - break; - - case DXGI_FORMAT_R16G16_FLOAT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case DXGI_FORMAT_R16G16_UNORM: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case DXGI_FORMAT_R16G16_UINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNSIGNED_INT16; - break; - - case DXGI_FORMAT_R16G16_SNORM: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SNORM_INT16; - break; - - case DXGI_FORMAT_R16G16_SINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SIGNED_INT16; - break; - - case DXGI_FORMAT_R32_TYPELESS: - fmt.image_channel_order = CL_R; - break; - - case DXGI_FORMAT_D32_FLOAT: - break; - - case DXGI_FORMAT_R32_FLOAT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case DXGI_FORMAT_R32_UINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNSIGNED_INT32; - break; - - case DXGI_FORMAT_R32_SINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SIGNED_INT32; - break; - - case DXGI_FORMAT_R24G8_TYPELESS: - fmt.image_channel_order = CL_RG; - break; - - case DXGI_FORMAT_D24_UNORM_S8_UINT: - break; - - case DXGI_FORMAT_R24_UNORM_X8_TYPELESS: - break; - - case DXGI_FORMAT_X24_TYPELESS_G8_UINT: - break; - - case DXGI_FORMAT_R9G9B9E5_SHAREDEXP: - break; - - case DXGI_FORMAT_R8G8_B8G8_UNORM: - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_G8R8_G8B8_UNORM: - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_B8G8R8A8_UNORM: - fmt.image_channel_order = CL_BGRA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_B8G8R8X8_UNORM: - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8G8_TYPELESS: - fmt.image_channel_order = CL_RG; - break; - - case DXGI_FORMAT_R8G8_UNORM: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8G8_UINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNSIGNED_INT8; - break; - - case DXGI_FORMAT_R8G8_SNORM: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SNORM_INT8; - break; - - case DXGI_FORMAT_R8G8_SINT: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_SIGNED_INT8; - break; - - case DXGI_FORMAT_R16_TYPELESS: - fmt.image_channel_order = CL_R; - break; - - case DXGI_FORMAT_R16_FLOAT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case DXGI_FORMAT_D16_UNORM: - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case DXGI_FORMAT_R16_UNORM: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case DXGI_FORMAT_R16_UINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNSIGNED_INT16; - break; - - case DXGI_FORMAT_R16_SNORM: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SNORM_INT16; - break; - - case DXGI_FORMAT_R16_SINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SIGNED_INT16; - break; - - case DXGI_FORMAT_B5G6R5_UNORM: - fmt.image_channel_data_type = CL_UNORM_SHORT_565; - break; - - case DXGI_FORMAT_B5G5R5A1_UNORM: - fmt.image_channel_order = CL_BGRA; - break; - - case DXGI_FORMAT_R8_TYPELESS: - fmt.image_channel_order = CL_R; - break; - - case DXGI_FORMAT_R8_UNORM: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R8_UINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNSIGNED_INT8; - break; - - case DXGI_FORMAT_R8_SNORM: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SNORM_INT8; - break; - - case DXGI_FORMAT_R8_SINT: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_SIGNED_INT8; - break; - - case DXGI_FORMAT_A8_UNORM: - fmt.image_channel_order = CL_A; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case DXGI_FORMAT_R1_UNORM: - fmt.image_channel_order = CL_R; - break; - - case DXGI_FORMAT_BC1_TYPELESS: - case DXGI_FORMAT_BC1_UNORM: - case DXGI_FORMAT_BC1_UNORM_SRGB: - case DXGI_FORMAT_BC2_TYPELESS: - case DXGI_FORMAT_BC2_UNORM: - case DXGI_FORMAT_BC2_UNORM_SRGB: - case DXGI_FORMAT_BC3_TYPELESS: - case DXGI_FORMAT_BC3_UNORM: - case DXGI_FORMAT_BC3_UNORM_SRGB: - case DXGI_FORMAT_BC4_TYPELESS: - case DXGI_FORMAT_BC4_UNORM: - case DXGI_FORMAT_BC4_SNORM: - case DXGI_FORMAT_BC5_TYPELESS: - case DXGI_FORMAT_BC5_UNORM: - case DXGI_FORMAT_BC5_SNORM: - break; - case DXGI_FORMAT_NV12: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNSIGNED_INT8; - if (plane == 1) { - fmt.image_channel_order = CL_RG; - } - break; - case DXGI_FORMAT_P010: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNSIGNED_INT16; - if (plane == 1) { - fmt.image_channel_order = CL_RG; - } - break; - default: - _ASSERT(FALSE); - break; - } - - return fmt; -} - -} // namespace amd - -#endif //_WIN32 diff --git a/src/hipamd/src/cl_d3d9.cpp b/src/hipamd/src/cl_d3d9.cpp deleted file mode 100644 index 3291e76c8e..0000000000 --- a/src/hipamd/src/cl_d3d9.cpp +++ /dev/null @@ -1,855 +0,0 @@ -/* Copyright (c) 2012-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifdef _WIN32 - -#include "top.hpp" - -#include "cl_d3d9_amd.hpp" -#include "platform/command.hpp" - -#include -#include - -#define D3DFMT_NV_12 static_cast(MAKEFOURCC('N', 'V', '1', '2')) -#define D3DFMT_P010 static_cast(MAKEFOURCC('P', '0', '1', '0')) -#define D3DFMT_YV_12 static_cast(MAKEFOURCC('Y', 'V', '1', '2')) -#define D3DFMT_YUY2 static_cast(MAKEFOURCC('Y', 'U', 'Y', '2')) - - -RUNTIME_ENTRY(cl_int, clGetDeviceIDsFromDX9MediaAdapterKHR, - (cl_platform_id platform, cl_uint num_media_adapters, - cl_dx9_media_adapter_type_khr* media_adapters_type, void* media_adapters, - cl_dx9_media_adapter_set_khr media_adapter_set, cl_uint num_entries, - cl_device_id* devices, cl_uint* num_devices)) { - cl_int errcode; - // Accept an array of DX9 devices here as the spec mention of array of num_media_adapters size. - IDirect3DDevice9Ex** d3d9_device = static_cast(media_adapters); - cl_device_id* gpu_devices = NULL; - cl_uint num_gpu_devices = 0; - static const bool VALIDATE_ONLY = true; - - if (platform != NULL && platform != AMD_PLATFORM) { - LogWarning("\"platrform\" is not a valid AMD platform"); - return CL_INVALID_PLATFORM; - } - // check if input parameter are correct - if ((num_media_adapters == 0) || (media_adapters_type == NULL) || (media_adapters == NULL) || - (media_adapter_set != CL_PREFERRED_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR && - media_adapter_set != CL_ALL_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR) || - (num_entries == 0 && devices != NULL)) { - return CL_INVALID_VALUE; - } - // Get GPU devices - errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 0, NULL, &num_gpu_devices); - if (errcode != CL_SUCCESS && errcode != CL_DEVICE_NOT_FOUND) { - return CL_INVALID_VALUE; - } - - if (!num_gpu_devices) { - *not_null(num_devices) = 0; - return CL_DEVICE_NOT_FOUND; - } - - switch (media_adapter_set) { - case CL_PREFERRED_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR: - case CL_ALL_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR: { - gpu_devices = new cl_device_id[num_gpu_devices]; - errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, num_gpu_devices, gpu_devices, NULL); - if (errcode != CL_SUCCESS) { - break; - } - - std::vector compatible_devices; - for (cl_uint i = 0; i < num_gpu_devices; ++i) { - cl_device_id device = gpu_devices[i]; - amd::Context::Flags context_flag; - amd::Context::DeviceFlagIdx devIdx; - switch (media_adapters_type[i]) { - case CL_ADAPTER_D3D9_KHR: - context_flag = amd::Context::Flags::D3D9DeviceKhr; - devIdx = amd::Context::DeviceFlagIdx::D3D9DeviceKhrIdx; - break; - case CL_ADAPTER_D3D9EX_KHR: - context_flag = amd::Context::Flags::D3D9DeviceEXKhr; - devIdx = amd::Context::DeviceFlagIdx::D3D9DeviceEXKhrIdx; - break; - case CL_ADAPTER_DXVA_KHR: - context_flag = amd::Context::Flags::D3D9DeviceVAKhr; - devIdx = amd::Context::DeviceFlagIdx::D3D9DeviceVAKhrIdx; - break; - } - - for (cl_uint j = 0; j < num_media_adapters; ++j) { - // Since there can be multiple DX9 adapters passed in the array we need to validate - // interopability with each. - void* external_device[amd::Context::DeviceFlagIdx::LastDeviceFlagIdx] = {}; - external_device[devIdx] = d3d9_device[j]; - - if (is_valid(device) && (media_adapters_type[j] == CL_ADAPTER_D3D9EX_KHR) && - as_amd(device)->bindExternalDevice(context_flag, external_device, NULL, - VALIDATE_ONLY)) { - compatible_devices.push_back(as_amd(device)); - } - } - } - if (compatible_devices.size() == 0) { - *not_null(num_devices) = 0; - errcode = CL_DEVICE_NOT_FOUND; - break; - } - - auto it = compatible_devices.cbegin(); - cl_uint compatible_count = std::min(num_entries, (cl_uint)compatible_devices.size()); - - while (compatible_count--) { - *devices++ = as_cl(*it++); - --num_entries; - } - while (num_entries--) { - *devices++ = (cl_device_id)0; - } - - *not_null(num_devices) = (cl_uint)compatible_devices.size(); - } break; - - default: - LogWarning("\"d3d9_device_set\" is invalid"); - errcode = CL_INVALID_VALUE; - } - - delete[] gpu_devices; - return errcode; -} -RUNTIME_EXIT - -RUNTIME_ENTRY_RET(cl_mem, clCreateFromDX9MediaSurfaceKHR, - (cl_context context, cl_mem_flags flags, - cl_dx9_media_adapter_type_khr adapter_type, void* surface_info, cl_uint plane, - cl_int* errcode_ret)) { - cl_mem clMemObj = NULL; - - cl_dx9_surface_info_khr* cl_surf_info = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - - if (!flags) flags = CL_MEM_READ_WRITE; - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - - if ((adapter_type != CL_ADAPTER_D3D9_KHR) && (adapter_type != CL_ADAPTER_D3D9EX_KHR) && - (adapter_type != CL_ADAPTER_DXVA_KHR)) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - return clMemObj; - } - - if (!surface_info) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("parameter \"pD3DResource\" is a NULL pointer"); - return clMemObj; - } - - cl_surf_info = (cl_dx9_surface_info_khr*)surface_info; - IDirect3DSurface9* pD3D9Resource = cl_surf_info->resource; - HANDLE shared_handle = cl_surf_info->shared_handle; - - if (!pD3D9Resource) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("parameter \"surface_info\" is a NULL pointer"); - return clMemObj; - } - - D3DSURFACE_DESC Desc; - pD3D9Resource->GetDesc(&Desc); - - if ((Desc.Format != D3DFMT_NV_12) && - (Desc.Format != D3DFMT_P010) && - (Desc.Format != D3DFMT_YV_12) && (plane != 0)) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("The plane has to be Zero if the surface format is non-planar !"); - return clMemObj; - } - - // Check for image support - const std::vector& devices = as_amd(context)->devices(); - bool supportPass = false; - bool sizePass = false; - for (const auto& it : devices) { - if (it->info().imageSupport_) { - supportPass = true; - } - } - if (!supportPass) { - *not_null(errcode_ret) = CL_INVALID_OPERATION; - LogWarning("there are no devices in context to support images"); - return (cl_mem)0; - } - // Verify the resource is a 2D image - return amd::clCreateImage2DFromD3D9ResourceAMD(*as_amd(context), flags, adapter_type, - cl_surf_info, plane, errcode_ret); -} -RUNTIME_EXIT - -RUNTIME_ENTRY(cl_int, clEnqueueAcquireDX9MediaSurfacesKHR, - (cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return amd::clEnqueueAcquireExtObjectsAMD(command_queue, num_objects, mem_objects, - num_events_in_wait_list, event_wait_list, event, - CL_COMMAND_ACQUIRE_DX9_MEDIA_SURFACES_KHR); -} -RUNTIME_EXIT - -RUNTIME_ENTRY(cl_int, clEnqueueReleaseDX9MediaSurfacesKHR, - (cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return amd::clEnqueueReleaseExtObjectsAMD(command_queue, num_objects, mem_objects, - num_events_in_wait_list, event_wait_list, event, - CL_COMMAND_RELEASE_DX9_MEDIA_SURFACES_KHR); -} -RUNTIME_EXIT - -// -// -// namespace amd -// -// -namespace amd { -/*! @} - * \addtogroup CL-D3D9 interop helper functions - * @{ - */ -// -// Class D3D9Object implementation -// -std::vector> D3D9Object::resources_; -Monitor D3D9Object::resLock_; - -// -// clCreateImage2DFromD3D9ResourceAMD -// -cl_mem clCreateImage2DFromD3D9ResourceAMD(Context& amdContext, cl_mem_flags flags, - cl_dx9_media_adapter_type_khr adapter_type, - cl_dx9_surface_info_khr* surface_info, cl_uint plane, - int* errcode_ret) { - cl_dx9_surface_info_khr* cl_surf_info = reinterpret_cast(surface_info); - IDirect3DSurface9* pD3D9Resource = cl_surf_info->resource; - HANDLE shared_handle = cl_surf_info->shared_handle; - - D3D9Object obj; - cl_int errcode = D3D9Object::initD3D9Object(amdContext, adapter_type, surface_info, plane, obj); - if (CL_SUCCESS != errcode) { - *not_null(errcode_ret) = errcode; - return (cl_mem)0; - } - - Image2DD3D9* pImage2DD3D9 = new (amdContext) Image2DD3D9(amdContext, flags, obj); - if (!pImage2DD3D9) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_mem)0; - } - if (!pImage2DD3D9->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImage2DD3D9->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImage2DD3D9); -} - -// -// Helper function SyncD3D9Objects -// -void SyncD3D9Objects(std::vector& memObjects) { - Memory*& mem = memObjects.front(); - if (!mem) { - LogWarning("\nNULL memory object\n"); - return; - } - InteropObject* interop = mem->getInteropObj(); - if (!interop) { - LogWarning("\nNULL interop object\n"); - return; - } - D3D9Object* d3d9Obj = interop->asD3D9Object(); - if (!d3d9Obj) { - LogWarning("\nNULL D3D9 object\n"); - return; - } - IDirect3DQuery9* query = d3d9Obj->getQuery(); - if (!query) { - LogWarning("\nNULL IDirect3DQuery9\n"); - return; - } - ScopedLock sl(d3d9Obj->getResLock()); - query->Issue(D3DISSUE_END); - BOOL data = FALSE; - while (S_OK != query->GetData(&data, sizeof(BOOL), D3DGETDATA_FLUSH)) { - } -} - -// -// Class D3D10Object implementation -// -size_t D3D9Object::getElementBytes(D3DFORMAT d3d9Format, cl_uint plane) { - size_t bytesPerPixel; - - switch (d3d9Format) { - case D3DFMT_UNKNOWN: - case D3DFMT_UYVY: - case D3DFMT_DXT1: - case D3DFMT_DXT2: - case D3DFMT_DXT3: - case D3DFMT_DXT4: - case D3DFMT_DXT5: - case D3DFMT_VERTEXDATA: - case D3DFMT_D32: - case D3DFMT_D15S1: - case D3DFMT_D24S8: - case D3DFMT_D24X8: - case D3DFMT_D24X4S4: - case D3DFMT_D16: - case D3DFMT_INDEX16: - case D3DFMT_INDEX32: - case D3DFMT_MULTI2_ARGB8: - case D3DFMT_CxV8U8: - // Less than 1 byte per pixel - needs special consideration - bytesPerPixel = 0; - break; - - case D3DFMT_R3G3B2: - case D3DFMT_P8: - case D3DFMT_A8: - case D3DFMT_L8: - case D3DFMT_A4L4: - bytesPerPixel = 1; - break; - - case D3DFMT_R16F: - case D3DFMT_R5G6B5: - case D3DFMT_X1R5G5B5: - case D3DFMT_A1R5G5B5: - case D3DFMT_A4R4G4B4: - case D3DFMT_A8R3G3B2: - case D3DFMT_X4R4G4B4: - case D3DFMT_A8P8: - case D3DFMT_A8L8: - case D3DFMT_V8U8: - case D3DFMT_L6V5U5: - case D3DFMT_D16_LOCKABLE: - case D3DFMT_L16: - bytesPerPixel = 2; - break; - - case D3DFMT_R8G8B8: - case D3DFMT_D24FS8: - bytesPerPixel = 3; - break; - - case D3DFMT_D32F_LOCKABLE: - case D3DFMT_A8R8G8B8: - case D3DFMT_R32F: - case D3DFMT_X8R8G8B8: - case D3DFMT_A2B10G10R10: - case D3DFMT_A8B8G8R8: - case D3DFMT_X8B8G8R8: - case D3DFMT_G16R16: - case D3DFMT_A2R10G10B10: - case D3DFMT_Q8W8V8U8: - case D3DFMT_X8L8V8U8: - case D3DFMT_V16U16: - case D3DFMT_A2W10V10U10: - case D3DFMT_R8G8_B8G8: - case D3DFMT_G8R8_G8B8: - case D3DFMT_G16R16F: - case D3DFMT_YUY2: - bytesPerPixel = 4; - break; - - case D3DFMT_G32R32F: - case D3DFMT_A16B16G16R16: - case D3DFMT_A16B16G16R16F: - case D3DFMT_Q16W16V16U16: - bytesPerPixel = 8; - break; - case D3DFMT_A32B32G32R32F: - bytesPerPixel = 16; - break; - //#if !defined(D3D_DISABLE_9EX) - // case D3DFMT_D32_LOCKABLE: - // case D3DFMT_S8_LOCKABLE: - //#endif // !D3D_DISABLE_9EX - case D3DFMT_NV_12: - if (plane == 0) { - bytesPerPixel = 1; - } else if (plane == 1) { - bytesPerPixel = 2; - } // plane != 0 or != 1 shouldn't happen here - break; - case D3DFMT_P010: - if (plane == 0) { - bytesPerPixel = 2; - } else if (plane == 1) { - bytesPerPixel = 4; - } // plane != 0 or != 1 shouldn't happen here - break; - case D3DFMT_YV_12: - bytesPerPixel = 1; - break; - - default: - bytesPerPixel = 0; - _ASSERT(FALSE); - break; - } - return bytesPerPixel; -} - -void setObjDesc(amd::D3D9ObjDesc_t& objDesc, D3DSURFACE_DESC& resDesc, cl_uint plane) { - objDesc.d3dPool_ = resDesc.Pool; - objDesc.resType_ = resDesc.Type; - objDesc.usage_ = resDesc.Usage; - objDesc.d3dFormat_ = resDesc.Format; - switch (resDesc.Format) { - case D3DFMT_NV_12: - case D3DFMT_P010: - objDesc.surfRect_.left = 0; - objDesc.surfRect_.top = 0; - if (plane == 0) { - objDesc.objSize_.Height = resDesc.Height; - objDesc.objSize_.Width = resDesc.Width; - objDesc.surfRect_.right = resDesc.Width; // resDesc.Width/2-1; - objDesc.surfRect_.bottom = 3 * resDesc.Height / 2; - ; // 3*resDesc.Height/2-1; - } else if (plane == 1) { - objDesc.objSize_.Height = resDesc.Height / 2; - objDesc.objSize_.Width = resDesc.Width / 2; - objDesc.surfRect_.right = resDesc.Width; // resDesc.Width/2-1; - objDesc.surfRect_.bottom = 3 * resDesc.Height / 2; - ; // 3*resDesc.Height/2-1; - } // plane != 0 or != 1 shouldn't happen here - break; - case D3DFMT_YV_12: - objDesc.surfRect_.left = 0; - if (plane == 0) { - objDesc.objSize_.Height = resDesc.Height; - objDesc.objSize_.Width = resDesc.Width; - objDesc.surfRect_.top = 0; - objDesc.surfRect_.right = resDesc.Width - 1; - objDesc.surfRect_.bottom = resDesc.Height - 1; - } else if (plane == 1) { - objDesc.objSize_.Height = resDesc.Height / 2; - objDesc.objSize_.Width = resDesc.Width / 2; - objDesc.surfRect_.top = resDesc.Height; - objDesc.surfRect_.right = resDesc.Width / 2 - 1; - objDesc.surfRect_.bottom = 3 * resDesc.Height / 2 - 1; - } else if (plane == 2) { - objDesc.objSize_.Height = resDesc.Height / 2; - objDesc.objSize_.Width = resDesc.Width / 2; - objDesc.surfRect_.top = 3 * resDesc.Height / 2; - objDesc.surfRect_.right = resDesc.Width / 2 - 1; - objDesc.surfRect_.bottom = 2 * resDesc.Height - 1; - } // plane > 0 or > 2 shouldn't happen here - break; - default: - objDesc.objSize_.Height = resDesc.Height; - objDesc.objSize_.Width = resDesc.Width; - objDesc.surfRect_.left = 0; - objDesc.surfRect_.top = 0; - objDesc.surfRect_.right = resDesc.Width - 1; - objDesc.surfRect_.bottom = resDesc.Height - 1; - if (resDesc.Format == D3DFMT_YUY2) { - objDesc.objSize_.Width >>= 1; - } - break; - } -} - -int D3D9Object::initD3D9Object(const Context& amdContext, - cl_dx9_media_adapter_type_khr adapter_type, - cl_dx9_surface_info_khr* cl_surf_info, cl_uint plane, - D3D9Object& obj) { - ScopedLock sl(resLock_); - - IDirect3DDevice9Ex* pDev9Ex = NULL; - cl_int errcode = CL_SUCCESS; - - // Check if this ressource has already been used for interop - IDirect3DSurface9* pD3D9res = cl_surf_info->resource; - HANDLE shared_handle = cl_surf_info->shared_handle; - - if ((adapter_type == CL_ADAPTER_D3D9_KHR) || (adapter_type == CL_ADAPTER_DXVA_KHR)) { - return CL_INVALID_DX9_MEDIA_ADAPTER_KHR; // Not supported yet - } - - for (const auto& it : resources_) { - if (it.first.surfInfo.resource == cl_surf_info->resource && it.first.surfPlane == plane) { - return CL_INVALID_D3D9_RESOURCE_KHR; - } - } - - HRESULT hr; - D3DQUERYTYPE desc = D3DQUERYTYPE_EVENT; - - D3DSURFACE_DESC resDesc; - if (D3D_OK != pD3D9res->GetDesc(&resDesc)) { - return CL_INVALID_D3D9_RESOURCE_KHR; - } - - hr = pD3D9res->GetContainer(IID_IDirect3DDevice9Ex, (void**)&pDev9Ex); - if (hr == D3D_OK) { - pDev9Ex->CreateQuery(desc, &(obj.pQuery_)); - } else { - return CL_INVALID_D3D9_RESOURCE_KHR; // d3d9ex should be supported - } - - obj.handleShared_ = shared_handle; - obj.surfPlane_ = plane; - obj.surfInfo_ = *cl_surf_info; - obj.adapterType_ = adapter_type; - - // Init defaults - setObjDesc(obj.objDescOrig_, resDesc, plane); - obj.objDesc_ = obj.objDescOrig_; - - // shared handle cases if the shared_handle is NULL - // first check if the format is NV12 or YV12, which we need special handling - if (NULL == shared_handle) { - bool found = false; - for (const auto& it : resources_) { - if (it.first.surfInfo.resource == cl_surf_info->resource && - it.first.surfPlane != plane) { - obj.handleShared_ = it.second.surfInfo.shared_handle; - obj.pD3D9Res_ = it.second.surfInfo.resource; - obj.pD3D9Res_->AddRef(); - obj.objDesc_ = obj.objDescOrig_; - found = true; - break; - } - } - if (!found) { - obj.handleShared_ = 0; - hr = pDev9Ex->CreateOffscreenPlainSurface(resDesc.Width, resDesc.Height, resDesc.Format, - resDesc.Pool, &obj.pD3D9Res_, &obj.handleShared_); - - if (D3D_OK != hr) { - errcode = CL_INVALID_D3D9_RESOURCE_KHR; - } - } - - // put the original info into the obj - obj.pD3D9ResOrig_ = pD3D9res; - obj.pD3D9ResOrig_->AddRef(); // addRef in case lost the resource - } else { - // Share the original resource - obj.pD3D9ResOrig_ = NULL; - obj.pD3D9Res_ = pD3D9res; - obj.pD3D9Res_->AddRef(); - } - - // Release the Ex interface - if (pDev9Ex) pDev9Ex->Release(); - - // Check for CL format compatibilty - if (obj.objDesc_.resType_ == D3DRTYPE_SURFACE) { - cl_image_format clFmt = obj.getCLFormatFromD3D9(obj.objDesc_.d3dFormat_, plane); - amd::Image::Format imageFormat(clFmt); - if (!imageFormat.isSupported(amdContext)) { - return CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - } - } - - TD3D9RESINFO d3d9ObjOri = {*cl_surf_info, plane}; - TD3D9RESINFO d3d9ObjShared = {{obj.pD3D9Res_, obj.handleShared_}, plane}; - - if (errcode == CL_SUCCESS) { - resources_.push_back({d3d9ObjOri, d3d9ObjShared}); - } - - return errcode; -} -cl_uint D3D9Object::getMiscFlag() { - switch (objDescOrig_.d3dFormat_) { - case D3DFMT_NV_12: - case D3DFMT_P010: - return 1; - break; - case D3DFMT_YV_12: - return 2; - break; - case D3DFMT_YUY2: - return 3; - break; - default: - return 0; - break; - } -} - -cl_image_format D3D9Object::getCLFormatFromD3D9() { - return getCLFormatFromD3D9(objDesc_.d3dFormat_, surfPlane_); -} - -cl_image_format D3D9Object::getCLFormatFromD3D9(D3DFORMAT d3d9Fmt, cl_uint plane) { - cl_image_format fmt; - - fmt.image_channel_order = 0; // CL_RGBA; - fmt.image_channel_data_type = 0; // CL_UNSIGNED_INT8; - - switch (d3d9Fmt) { - case D3DFMT_R32F: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case D3DFMT_R16F: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case D3DFMT_L16: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case D3DFMT_A8: - fmt.image_channel_order = CL_A; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case D3DFMT_L8: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case D3DFMT_G32R32F: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case D3DFMT_G16R16F: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case D3DFMT_G16R16: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case D3DFMT_A8L8: - fmt.image_channel_order = CL_RG; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case D3DFMT_A32B32G32R32F: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_FLOAT; - break; - - case D3DFMT_A16B16G16R16F: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_HALF_FLOAT; - break; - - case D3DFMT_A16B16G16R16: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT16; - break; - - case D3DFMT_A8B8G8R8: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case D3DFMT_X8B8G8R8: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case D3DFMT_A8R8G8B8: - fmt.image_channel_order = CL_BGRA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - - case D3DFMT_X8R8G8B8: - fmt.image_channel_order = CL_BGRA; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - case D3DFMT_NV_12: - fmt.image_channel_data_type = CL_UNORM_INT8; - if (plane == 0) { - fmt.image_channel_order = CL_R; - } else if (plane == 1) { - fmt.image_channel_order = CL_RG; - } - break; - case D3DFMT_P010: - fmt.image_channel_data_type = CL_UNORM_INT16; - if (plane == 0) { - fmt.image_channel_order = CL_R; - } else if (plane == 1) { - fmt.image_channel_order = CL_RG; - } - break; - case D3DFMT_YV_12: - fmt.image_channel_order = CL_R; - fmt.image_channel_data_type = CL_UNORM_INT8; - break; - case D3DFMT_YUY2: - fmt.image_channel_order = CL_RGBA; - fmt.image_channel_data_type = CL_UNSIGNED_INT8; - break; - case D3DFMT_UNKNOWN: - case D3DFMT_R8G8B8: - case D3DFMT_R5G6B5: - case D3DFMT_X1R5G5B5: - case D3DFMT_A1R5G5B5: - case D3DFMT_A4R4G4B4: - case D3DFMT_R3G3B2: - case D3DFMT_A8R3G3B2: - case D3DFMT_X4R4G4B4: - case D3DFMT_A2B10G10R10: - case D3DFMT_A2R10G10B10: - case D3DFMT_A8P8: - case D3DFMT_P8: - case D3DFMT_A4L4: - case D3DFMT_V8U8: - case D3DFMT_L6V5U5: - case D3DFMT_X8L8V8U8: - case D3DFMT_Q8W8V8U8: - case D3DFMT_V16U16: - case D3DFMT_A2W10V10U10: - case D3DFMT_UYVY: - case D3DFMT_R8G8_B8G8: - case D3DFMT_G8R8_G8B8: - case D3DFMT_DXT1: - case D3DFMT_DXT2: - case D3DFMT_DXT3: - case D3DFMT_DXT4: - case D3DFMT_DXT5: - case D3DFMT_D16_LOCKABLE: - case D3DFMT_D32: - case D3DFMT_D15S1: - case D3DFMT_D24S8: - case D3DFMT_D24X8: - case D3DFMT_D24X4S4: - case D3DFMT_D16: - case D3DFMT_D32F_LOCKABLE: - case D3DFMT_D24FS8: - //#if !defined(D3D_DISABLE_9EX) - case D3DFMT_D32_LOCKABLE: - case D3DFMT_S8_LOCKABLE: - //#endif // !D3D_DISABLE_9EX - case D3DFMT_VERTEXDATA: - case D3DFMT_INDEX16: - case D3DFMT_INDEX32: - case D3DFMT_Q16W16V16U16: - case D3DFMT_MULTI2_ARGB8: - case D3DFMT_CxV8U8: - //#if !defined(D3D_DISABLE_9EX) - case D3DFMT_A1: - case D3DFMT_A2B10G10R10_XR_BIAS: - case D3DFMT_BINARYBUFFER: - _ASSERT(FALSE); // NOT SURPPORTED - break; - //#endif // !D3D_DISABLE_9EX - default: - _ASSERT(FALSE); - break; - } - - return fmt; -} - -bool D3D9Object::copyOrigToShared() { - // Don't copy if there is no orig - if (NULL == getD3D9ResOrig()) return true; - - IDirect3DDevice9Ex* d3dDev; - HRESULT hr; - ScopedLock sl(getResLock()); - - IDirect3DSurface9* srcSurf = getD3D9ResOrig(); - IDirect3DSurface9* dstSurf = getD3D9Resource(); - - hr = getD3D9Resource()->GetContainer(IID_IDirect3DDevice9Ex, (void**)&d3dDev); - if (hr != D3D_OK || !d3dDev) { - LogError("\nCannot get D3D9 device from D3D9 surface\n"); - return false; - } - - hr = d3dDev->StretchRect(srcSurf, NULL, dstSurf, NULL, D3DTEXF_NONE); - if (hr != D3D_OK) { - LogError("\ncopy original surface to shared surface failed\n"); - return false; - } - // Flush D3D queues and make sure D3D stuff is finished - pQuery_->Issue(D3DISSUE_END); - BOOL data; - while ((D3D_OK != pQuery_->GetData(&data, sizeof(BOOL), D3DGETDATA_FLUSH)) && (data != TRUE)) { - } - - if (d3dDev) d3dDev->Release(); - return true; -} - -bool D3D9Object::copySharedToOrig() { - // Don't copy if there is no orig - if (NULL == getD3D9ResOrig()) return true; - - IDirect3DDevice9Ex* d3dDev; - HRESULT hr; - ScopedLock sl(getResLock()); - - hr = getD3D9Resource()->GetContainer(IID_IDirect3DDevice9Ex, (void**)&d3dDev); - if (hr != D3D_OK || !d3dDev) { - LogError("\nCannot get D3D9 device from D3D9 surface\n"); - return false; - } - - hr = d3dDev->StretchRect(getD3D9Resource(), NULL, getD3D9ResOrig(), NULL, D3DTEXF_NONE); - if (hr != D3D_OK) { - LogError("\ncopy shared surface to original surface failed\n"); - return false; - } - - if (d3dDev) d3dDev->Release(); - return true; -} - -void Image2DD3D9::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(Image2DD3D9)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -} // namespace amd - -#endif //_WIN32 diff --git a/src/hipamd/src/cl_gl.cpp b/src/hipamd/src/cl_gl.cpp deleted file mode 100644 index 6a32525da2..0000000000 --- a/src/hipamd/src/cl_gl.cpp +++ /dev/null @@ -1,2432 +0,0 @@ -/* Copyright (c) 2010-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "top.hpp" - -#ifdef _WIN32 -#include -#include -#include -// This is necessary since there are common GL/D3D10 functions -#include "cl_d3d9_amd.hpp" -#include "cl_d3d10_amd.hpp" -#include "cl_d3d11_amd.hpp" -#endif //_WIN32 - -#include -#include - -#include -#include -#include - -#include "cl_common.hpp" -#include "cl_gl_amd.hpp" - -#include "device/device.hpp" - -/* The pixel internal format for DOPP texture defined in gl_enum.h */ -#define GL_BGR8_ATI 0x8083 -#define GL_BGRA8_ATI 0x8088 - -#include -#include - - -/*! \addtogroup API - * @{ - * - * \addtogroup CL_GL_Interops - * - * This section discusses OpenCL functions that allow applications to - * use OpenGL buffer/texture/render-buffer objects as OpenCL memory - * objects. This allows efficient sharing of data between these OpenCL - * and OpenGL. The OpenCL API can be used to execute kernels that read - * and/or write memory objects that are also an OpenGL buffer object - * or a texture. An OpenCL image object can be created from an OpenGL - * texture or renderbuffer object. An OpenCL buffer object can be - * created from an OpenGL buffer object. An OpenCL memory object can - * be created from an OpenGL texture/buffer/render-buffer object or - * the default system provided framebuffer if any only if the OpenCL - * clContext has been created from a GL clContext. OpenGL contexts are - * created using platform specific APIs (EGL, CGL, WGL, GLX are some - * of the platform specific APIs that allow applications to create GL - * contexts). The appropriate platform API (such as EGL, CGL, WGL, - * GLX) will be extended to allow a CL clContext to be created from a - * GL clContext. Creating an OpenCL memory object from the default - * system provided framebuffer will also require an appropriate - * extension to the platform API. Refer to the appropriate platform - * API documentation to understand how to create a CL clContext from a - * GL clContext and creating a CL memory object from the default - * system provided framebuffer. - * - * @{ - * - * \addtogroup clCreateFromGLBuffer - * - * @{ - */ - -/*! \brief Creates an OpenCL buffer object from an OpenGL buffer object. - * - * \param clContext is a valid OpenCL clContext created from an OpenGL clContext. - * - * \param clFlags is a bit-field that is used to specify usage information. Only - * CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE can be used. - * - * \param glBufferName is a GL buffer object name. The GL buffer - * object must have a data store created though it does not need to - * be initialized. The size of the data store will be used to - * determine the size of the CL buffer object. - * - * \param pCpuMem is a pointer to the buffer data that may already be - * allocated by the application. The size of the buffer that pCpuMem points - * to must be >= \a size bytes. Passing in a pointer to an already allocated - * buffer on the host and using it as a buffer object allows applications to - * share data efficiently with kernels and the host. - * - * \param errcode_ret will return an appropriate error code. If errcode_ret - * is NULL, no error code is returned. - * - * \return valid non-zero OpenCL buffer object and errcode_ret is set - * to CL_SUCCESS if the buffer object is created successfully. It - * returns a NULL value with one of the following error values - * returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a clContext is not a valid clContext. - * - CL_INVALID_VALUE if values specified in \a clFlags are not valid. - * - CL_INVALID_GL_OBJECT if glBufferName is not a GL buffer object or is a - * GL buffer object but does not have a data store created. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.0r29 - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromGLBuffer, - (cl_context context, cl_mem_flags flags, GLuint bufobj, cl_int* errcode_ret)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - - return (amd::clCreateFromGLBufferAMD(*as_amd(context), flags, bufobj, errcode_ret)); -} -RUNTIME_EXIT - -/*! \brief creates the following: - * - an OpenCL 2D image object from an OpenGL 2D texture object - * or a single face of an OpenGL cubemap texture object, - * - an OpenCL 2D image array object from an OpenGL 2D texture array object, - * - an OpenCL 1D image object from an OpenGL 1D texture object, - * - an OpenCL 1D image buffer object from an OpenGL texture buffer object, - * - an OpenCL 1D image array object from an OpenGL 1D texture array object, - * - an OpenCL 3D image object from an OpenGL 3D texture object. - * - * \param clContext is a valid OpenCL clContext created from an OpenGL clContext. - * - * \param clFlags is a bit-field that is used to specify usage information. - * Only CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE values - * can be used. - * - * \param texture_target must be GL_TEXTURE_1D, GL_TEXTURE_1D_ARRAY, - * GL_TEXTURE_BUFFER, GL_TEXTURE_2D_ARRAY, GL_TEXTURE_3D, - * GL_TEXTURE_2D, GL_TEXTURE_CUBE_MAP_POSITIVE_X, - * GL_TEXTURE_CUBE_MAP_POSITIVE_Y, GL_TEXTURE_CUBE_MAP_POSITIVE_Z, - * GL_TEXTURE_CUBE_MAP_NEGATIVE_X, GL_TEXTURE_CUBE_MAP_NEGATIVE_Y, - * GL_TEXTURE_CUBE_MAP_NEGATIVE_Z or GL_TEXTURE_RECTANGLE_ARB. - * - * \param miplevel is the mipmap level to be used. If \a texture_target - * is GL_TEXTURE_BUFFER, \a miplevel must be 0. - * - * \param texture is a GL 1D, 2D, 3D, 1D array, 2D array, cubemap, - * rectangle or buffer texture object. - * The texture object must be a complete texture as per - * OpenGL rules on texture completeness. The texture format and dimensions - * defined by OpenGL for the specified miplevel of the texture will be - * used to create the OpenCL image memory object. Only GL texture formats - * that map to appropriate image channel order and data type can be used - * to create the the OpenCL image memory object. - * - * \param errcode_ret will return an appropriate error code. If \a - * errcode_ret is NULL, no error code is returned. - * - * \return A valid non-zero OpenCL image object and \a errcode_ret is set to - * CL_SUCCESS if the image object is created successfully. It returns a NULL value - * with one of the following error values returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a clContext is not a valid clContext or was not - * created from a GL clContext. - * - CL_INVALID_VALUE if values specified in \a clFlags are not valid. - * - CL_INVALID_MIP_LEVEL if \a miplevel is not a valid mip-level for \a texture. - * - CL_INVALID_GL_OBJECT if \a texture is not an appropriate GL 2D texture, - * cubemap or texture rectangle. - * - CL_INVALID_IMAGE_FORMAT_DESCRIPTOR if the OpenGL texture format does not - * map to an appropriate OpenCL image format. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.2r07 - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromGLTexture, - (cl_context context, cl_mem_flags flags, GLenum texture_target, GLint miplevel, - GLuint texture, cl_int* errcode_ret)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - - const std::vector& devices = as_amd(context)->devices(); - bool supportPass = false; - bool sizePass = false; - for (const auto& it : devices) { - if (it->info().imageSupport_) { - supportPass = true; - } - } - if (!supportPass) { - *not_null(errcode_ret) = CL_INVALID_OPERATION; - LogWarning("there are no devices in context to support images"); - return static_cast(0); - } - - return amd::clCreateFromGLTextureAMD(*as_amd(context), flags, texture_target, miplevel, texture, - errcode_ret); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromGLTexture2D - * @{ - */ - -/*! \brief Create an OpenCL 2D image object from an OpenGL 2D texture object. - * - * \param clContext is a valid OpenCL clContext created from an OpenGL clContext. - * - * \param clFlags is a bit-field that is used to specify usage information. - * Only CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE values - * can be used. - * - * \param target must be GL_TEXTURE_2D, GL_TEXTURE_CUBE_MAP_POSITIVE_X, - * GL_TEXTURE_CUBE_MAP_POSITIVE_Y, GL_TEXTURE_CUBE_MAP_POSITIVE_Z, - * GL_TEXTURE_CUBE_MAP_NEGATIVE_X, GL_TEXTURE_CUBE_MAP_NEGATIVE_Y, - * GL_TEXTURE_CUBE_MAP_NEGATIVE_Z or GL_TEXTURE_RECTANGLE_ARB. - * - * \param miplevel is the mipmap level to be used. - * - * \param texture is a GL 2D texture, cubemap or texture rectangle - * object name. The texture object must be a complete texture as per - * OpenGL rules on texture completeness. The \a texture format and - * dimensions specified using appropriate glTexImage2D call for \a - * miplevel will be used to create the 2D image object. Only GL - * texture formats that map to appropriate image channel order and - * data type can be used to create the 2D image object. - * - * \param errcode_ret will return an appropriate error code. If \a - * errcode_ret is NULL, no error code is returned. - * - * \return A valid non-zero OpenCL image object and \a errcode_ret is set to - * CL_SUCCESS if the image object is created successfully. It returns a NULL value - * with one of the following error values returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a clContext is not a valid clContext or was not - * created from a GL clContext. - * - CL_INVALID_VALUE if values specified in \a clFlags are not valid. - * - CL_INVALID_MIP_LEVEL if \a miplevel is not a valid mip-level for \a texture. - * - CL_INVALID_GL_OBJECT if \a texture is not an appropriate GL 2D texture, - * cubemap or texture rectangle. - * - CL_INVALID_IMAGE_FORMAT_DESCRIPTOR if the OpenGL texture format does not - * map to an appropriate OpenCL image format. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.0r29 - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromGLTexture2D, - (cl_context context, cl_mem_flags flags, GLenum target, GLint miplevel, - GLuint texture, cl_int* errcode_ret)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - - const std::vector& devices = as_amd(context)->devices(); - bool supportPass = false; - bool sizePass = false; - for (const auto& it : devices) { - if (it->info().imageSupport_) { - supportPass = true; - } - } - if (!supportPass) { - *not_null(errcode_ret) = CL_INVALID_OPERATION; - LogWarning("there are no devices in context to support images"); - return static_cast(0); - } - - return amd::clCreateFromGLTextureAMD(*as_amd(context), flags, target, miplevel, texture, - errcode_ret); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromGLTexture3D - * @{ - */ - -/*! \brief Create an OpenCL 3D image object from an OpenGL 3D texture object. - * - * \param clContext is a valid OpenCL clContext created from an OpenGL clContext. - * - * \param clFlags is a bit-field that is used to specify usage information. - * Only CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE values - * can be used. - * - * \param target must be GL_TEXTURE_3D. - * - * \param miplevel is the mipmap level to be used. - * - * \param texture is a GL 3D texture object [name]. - * The texture object must be a complete texture as per OpenGL rules on texture - * completeness. The \a texture format and dimensions specified using appropriate - * glTexImage3D call for \a miplevel will be used to create the 3D image object. - * Only GL texture formats that map to appropriate image channel order and - * data type can be used to create the 3D image object. - * - * \param errcode_ret will return an appropriate error code. If \a errcode_ret - * is NULL, no error code is returned. - * - * \return A valid non-zero OpenCL image object and \a errcode_ret is set to - * CL_SUCCESS if the image object is created successfully. It returns a NULL value - * with one of the following error values returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a clContext is not a valid clContext or was not - * created from a GL clContext. - * - CL_INVALID_VALUE if values specified in \a clFlags are not valid. - * - CL_INVALID_MIP_LEVEL if \a miplevel is not a valid mip-level for \a texture. - * - CL_INVALID_GL_OBJECT if \a texture is not an GL 3D texture. - * - CL_INVALID_IMAGE_FORMAT_DESCRIPTOR if the OpenGL texture format does not - * map to an appropriate OpenCL image format. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.0r29 - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromGLTexture3D, - (cl_context context, cl_mem_flags flags, GLenum target, GLint miplevel, - GLuint texture, cl_int* errcode_ret)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - - const std::vector& devices = as_amd(context)->devices(); - bool supportPass = false; - bool sizePass = false; - for (const auto& it : devices) { - if (it->info().imageSupport_) { - supportPass = true; - } - } - if (!supportPass) { - *not_null(errcode_ret) = CL_INVALID_OPERATION; - LogWarning("there are no devices in context to support images"); - return static_cast(0); - } - - return amd::clCreateFromGLTextureAMD(*as_amd(context), flags, target, miplevel, texture, - errcode_ret); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clCreateFromGLRenderbuffer - * @{ - */ - -/*! \brief Create an OpenCL 2D image object from an OpenGL renderbuffer object. - * - * \param clContext is a valid OpenCL clContext created from an OpenGL clContext. - * - * \param clFlags is a bit-field that is used to specify usage information. - * Only CL_MEM_READ_ONLY, CL_MEM_WRITE_ONLY and CL_MEM_READ_WRITE values - * can be used. - * - * \param renderbuffer is a GL renderbuffer object name. The renderbuffer - * storage must be specified before the image object can be created. Only - * GL renderbuffer formats that map to appropriate image channel order and - * data type can be used to create the 2D image object. - * - * \param errcode_ret will return an appropriate error code. If \a errcode_ret - * is NULL, no error code is returned. - * - * \return A valid non-zero OpenCL image object and \a errcode_ret is set - * to CL_SUCCESS if the image object is created successfully. It returns a - * NULL value with one of the following error values returned in \a errcode_ret: - * - CL_INVALID_CONTEXT if \a clContext is not a valid clContext or was not - * created from a GL clContext. - * - CL_INVALID_VALUE if values specified in \a clFlags are not valid. - * - CL_INVALID_GL_OBJECT if \a renderbuffer is not an GL renderbuffer object. - * - CL_INVALID_IMAGE_FORMAT_DESCRIPTOR if the OpenGL renderbuffer format - * does not map to an appropriate OpenCL image format. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources required - * by the runtime. - * - * \version 1.0r29 - */ -RUNTIME_ENTRY_RET(cl_mem, clCreateFromGLRenderbuffer, (cl_context context, cl_mem_flags flags, - GLuint renderbuffer, cl_int* errcode_ret)) { - cl_mem clMemObj = NULL; - - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return clMemObj; - } - - if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) || - ((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) || - ((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid parameter \"flags\""); - return clMemObj; - } - - return (amd::clCreateFromGLRenderbufferAMD(*as_amd(context), flags, renderbuffer, errcode_ret)); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clGetGLObjectInfo - * @{ - */ - -/*! \brief Query GL object type from a CL memory object. - * - * \param memobj [is a valid cl_mem object created from a GL object]. - * - * \param gl_object_type returns the type of GL object attached to memobj - * and can be CL_GL_OBJECT_BUFFER, CL_GL_OBJECT_TEXTURE2D, - * CL_GL_OBJECT_TEXTURE_RECTANGLE, CL_GL_OBJECT_TEXTURE3D, or - * CL_GL_OBJECT_RENDERBUFFER. If \a gl_object_type is NULL, it is ignored. - * - * \param gl_object_name returns the GL object name used to create memobj. - * If \a gl_object_name is NULL, it is ignored. - * - * \return One of the following values is returned: - * - CL_SUCCESS if the call was executed successfully. - * - CL_INVALID_MEM_OBJECT if \a memobj is not a valid OpenCL memory object. - * - CL_INVALID_GL_OBJECT if there is no GL object associated with \a memobj. - * - * \version 1.0r29 - */ -RUNTIME_ENTRY(cl_int, clGetGLObjectInfo, - (cl_mem memobj, cl_gl_object_type* gl_object_type, GLuint* gl_object_name)) { - if (!is_valid(memobj)) { - LogWarning("\"memobj\" is not a valid cl_mem object"); - return CL_INVALID_MEM_OBJECT; - } - - amd::InteropObject* interop = as_amd(memobj)->getInteropObj(); - if (NULL == interop) { - LogWarning("CL object \"memobj\" is not created from GL object"); - return CL_INVALID_GL_OBJECT; - } - - amd::GLObject* glObject = interop->asGLObject(); - if (NULL == glObject) { - LogWarning("CL object \"memobj\" is not created from GL object"); - return CL_INVALID_GL_OBJECT; - } - - cl_int result; - - cl_gl_object_type clGLType = glObject->getCLGLObjectType(); - result = amd::clGetInfo(clGLType, sizeof(cl_gl_object_type), gl_object_type, NULL); - - GLuint glName = glObject->getGLName(); - result |= amd::clGetInfo(glName, sizeof(GLuint), gl_object_name, NULL); - - return result; -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clGetGLTextureInfo - * @{ - */ - -/*! \brief Query additional information about the GL texture object associated - * with \a memobj. - * - * \param memobj [is a valid cl_mem object created from a GL object]. - * - * \param param_name specifies what additional information about the GL - * texture object associated with \a memobj to query: - * - CL_GL_TEXTURE_TARGET (GLenum) to query the \a target argument specified - * in clCreateGLTexture2D or clCreateGLTexture3D calls. - * - CL_GL_MIPMAP_LEVEL (GLint) to query the \a miplevel argument specified - * in clCreateGLTexture2D or clCreateGLTexture3D calls. - * - * \param param_value is a pointer to memory where the appropriate result - * being queried is returned. If \a param_value is NULL, it is ignored. - * - * \param param_value_size is used to specify the size in bytes of memory - * pointed to by \a param_value. This size must be >= size of return type as - * described for \a param_name argumnet (GLenum or GLint). - * \a param_value_size_ret returns the actual size in bytes of data copied to - * \a param_value. If \a param_value_size_ret is NULL, it is ignored - * - * \return One of the following values is returned: - * - CL_SUCCESS if the function is executed successfully. - * - CL_INVALID_MEM_OBJECT if \a memobj is not a valid OpenCL memory object. - * - CL_INVALID_GL_OBJECT if there is no GL texture object (2D or 3D texture) - * associated with \a memobj. - * - CL_INVALID_VALUE if \a param_name is not valid, or if size in bytes - * specified by \a param_value_size is < size of return type required by - * \a param_name and \a param_value is not NULL, or if \a param_value and - * \a param_value_size_ret are NULL. - * - * \version 1.0r29 - */ -RUNTIME_ENTRY(cl_int, clGetGLTextureInfo, - (cl_mem memobj, cl_gl_texture_info param_name, size_t param_value_size, - void* param_value, size_t* param_value_size_ret)) { - if (!is_valid(memobj)) { - LogWarning("\"memobj\" is not a valid cl_mem object"); - return CL_INVALID_MEM_OBJECT; - } - amd::InteropObject* interop = as_amd(memobj)->getInteropObj(); - if (NULL == interop) { - LogWarning("CL object \"memobj\" is not created from GL object"); - return CL_INVALID_GL_OBJECT; - } - amd::GLObject* glObject = interop->asGLObject(); - if ((NULL == glObject) || (NULL != glObject->asBufferGL())) { - LogWarning("CL object \"memobj\" is not created from GL texture"); - return CL_INVALID_GL_OBJECT; - } - - switch (param_name) { - case CL_GL_TEXTURE_TARGET: { - GLenum glTarget = glObject->getGLTarget(); - if (glTarget == GL_TEXTURE_CUBE_MAP) { - glTarget = glObject->getCubemapFace(); - } - return amd::clGetInfo(glTarget, param_value_size, param_value, param_value_size_ret); - } - case CL_GL_MIPMAP_LEVEL: { - GLint mipLevel = glObject->getGLMipLevel(); - return amd::clGetInfo(mipLevel, param_value_size, param_value, param_value_size_ret); - } - case CL_GL_NUM_SAMPLES: { - GLsizei numSamples = glObject->getNumSamples(); - return amd::clGetInfo(numSamples, param_value_size, param_value, param_value_size_ret); - } - default: - LogWarning("Unknown param_name in clGetGLTextureInfoAMD"); - break; - } - - return CL_INVALID_VALUE; -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clEnqueueAcquireExtObjects - * @{ - */ - -/*! \brief Acquire OpenCL memory objects that have been created from external - * objects (OpenGL, D3D). - * - * \param command_queue is a valid command-queue. - * - * \param num_objects is the number of memory objects to be acquired - * in \a mem_objects. - * - * \param mem_objects is a pointer to a list of CL memory objects that refer - * to a GL object (buffer/texture/renderbuffer objects or the framebuffer). - * - * \param event_wait_list specify [is a pointer to] events that need to - * complete before this particular command can be executed. - * If \a event_wait_list is NULL, then this particular command does not wait - * on any event to complete. If \a event_wait_list is NULL, - * \a num_events_in_wait_list must be 0. If \a event_wait_list is not NULL, - * the list of events pointed to by \a event_wait_list must be valid and - * \a num_events_in_wait_list must be greater than 0. The events specified in - * \a event_wait_list act as synchronization points. - * - * \param num_events_in_wait_list specify the number of events in - * \a event_wait_list. It must be 0 if \a event_wait_list is NULL. It must be - * greater than 0 if \a event_wait_list is not NULL. - * - * \param event returns an event object that identifies this particular - * command and can be used to query or queue a wait for this particular - * command to complete. \a event can be NULL in which case it will not be - * possible for the application to query the status of this command or queue a - * wait for this command to complete. - * - * \return One of the following values is returned: - * - CL_SUCCESS if the function is executed successfully. - * - CL_SUCCESS if \a num_objects is 0 and \a mem_objects is NULL; the - * function does nothing. - * - CL_INVALID_VALUE if \a num_objects is zero and \a mem_objects is not a - * NULL value or if \a num_objects > 0 and \a mem_objects is NULL. - * - CL_INVALID_MEM_OBJECT if memory objects in \a mem_objects are not valid - * OpenCL memory objects. - * - CL_INVALID_COMMAND_QUEUE if \a command_queue is not a valid command-queue. - * - CL_INVALID_CONTEXT if clContext associated with \a command_queue was not - * created from an OpenGL clContext. - * - CL_INVALID_GL_OBJECT if memory objects in \a mem_objects have not been - * created from a GL object(s). - * - CL_INVALID_EVENT_WAIT_LIST if \a event_wait_list is NULL and - * \a num_events_in_wait_list > 0, or \a event_wait_list is not NULL and - * \a num_events_in_wait_list is 0, or if event objects in \a event_wait_list - * are not valid events. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources - * required by the OpenCL implementation on the host. - * - * \version 1.0r29 - */ -RUNTIME_ENTRY(cl_int, clEnqueueAcquireGLObjects, - (cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return amd::clEnqueueAcquireExtObjectsAMD(command_queue, num_objects, mem_objects, - num_events_in_wait_list, event_wait_list, event, - CL_COMMAND_ACQUIRE_GL_OBJECTS); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clEnqueueReleaseGLObjects - * @{ - */ - -/*! \brief Release OpenCL memory objects that have been created from OpenGL - * objects. - * - * \param command_queue is a valid command-queue [which is associated with the - * OpenCL clContext releasing the OpenGL objects]. - * - * \param num_objects is the number of memory objects to be released - * in \a mem_objects. - * - * \param mem_objects is a pointer to a list of CL memory objects that refer - * to a GL object (buffer/texture/renderbuffer objects or the framebuffer). - * - * \param event_wait_list specify [is a pointer to] events that need to - * complete before this particular command can be executed. - * If \a event_wait_list is NULL, then this particular command does not wait - * on any event to complete. If \a event_wait_list is NULL, - * \a num_events_in_wait_list must be 0. If \a event_wait_list is not NULL, - * the list of events pointed to by \a event_wait_list must be valid and - * \a num_events_in_wait_list must be greater than 0. The events specified in - * \a event_wait_list act as synchronization points. - * - * \param num_events_in_wait_list specify the number of events in - * \a event_wait_list. It must be 0 if \a event_wait_list is NULL. It must be - * greater than 0 if \a event_wait_list is not NULL. - * - * \param event returns an event object that identifies this particular - * command and can be used to query or queue a wait for this particular - * command to complete. \a event can be NULL in which case it will not be - * possible for the application to query the status of this command or queue a - * wait for this command to complete. - * - * \return One of the following values is returned: - * - CL_SUCCESS if the function is executed successfully. - * - CL_SUCCESS if \a num_objects is 0 and \a mem_objects is NULL; the - * function does nothing. - * - CL_INVALID_VALUE if \a num_objects is zero and \a mem_objects is not a - * NULL value or if \a num_objects > 0 and \a mem_objects is NULL. - * - CL_INVALID_MEM_OBJECT if memory objects in \a mem_objects are not valid - * OpenCL memory objects. - * - CL_INVALID_COMMAND_QUEUE if \a command_queue is not a valid command-queue. - * - CL_INVALID_CONTEXT if clContext associated with \a command_queue was not - * created from an OpenGL clContext. - * - CL_INVALID_GL_OBJECT if memory objects in \a mem_objects have not been - * created from a GL object(s). - * - CL_INVALID_EVENT_WAIT_LIST if \a event_wait_list is NULL and - * \a num_events_in_wait_list > 0, or \a event_wait_list is not NULL and - * \a num_events_in_wait_list is 0, or if event objects in \a event_wait_list - * are not valid events. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources - * required by the OpenCL implementation on the host. - * - * \version 1.0r29 - */ -RUNTIME_ENTRY(cl_int, clEnqueueReleaseGLObjects, - (cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return amd::clEnqueueReleaseExtObjectsAMD(command_queue, num_objects, mem_objects, - num_events_in_wait_list, event_wait_list, event, - CL_COMMAND_RELEASE_GL_OBJECTS); -} -RUNTIME_EXIT - -/*! @} -* \addtogroup clCreateEventFromGLsyncKHR -* @{ -*/ - -/*! \brief Creates an event object linked to an OpenGL sync object. -* Completion of such an event object is equivalent to waiting for completion -* of the fence command associated with the linked GL sync object. -* -* \param context is valid OpenCL context created from an OpenGL context -* or share group, using the cl_khr_gl_sharing extension. -* -* \param sync is the 'name' of a sync object in the GL share group associated -* with context. -* -* \param errcode_ret Returns an appropriate error code as described below. -* If errcode_ret is NULL, no error code is returned. -* -* \return a valid OpenCL event object and errcode_ret is set to CL_SUCCESS -* if the event object is created successfully.Otherwise, it returns a NULL -* value with one of the following error values returned in errcode_ret: -* - CL_INVALID_CONTEXT if context is not a valid context or was not created -* from a GL context. -* - CL_INVALID_GL_OBJECT if sync is not the name of a sync object in the -* GL share group associated with context. -* -* \version 1.1 -*/ - -RUNTIME_ENTRY_RET(cl_event, clCreateEventFromGLsyncKHR, - (cl_context context, cl_GLsync clGLsync, cl_int* errcode_ret)) { - // create event of fence sync type - amd::ClGlEvent* clglEvent = new amd::ClGlEvent(*as_amd(context)); - clglEvent->context().glenv()->glFlush_(); - // initially set the status of fence as queued - clglEvent->setStatus(CL_SUBMITTED); - // store GLsync id of the fence in event in order to associate them together - clglEvent->setData(clGLsync); - amd::Event* evt = dynamic_cast(clglEvent); - evt->retain(); - return as_cl(evt); -} -RUNTIME_EXIT - -/*! @} - * \addtogroup clGetGLContextInfoKHR - * @{ - */ - -/*! \brief This f-n is defined in CL extension cl_khr_gl_sharing and serves - * the purpose of quering current device and all devices that support - * CL-GL interoperability. - * - * \param properties points to an , which is a array of - * ordered pairs terminated with zero. If an - * attribute is not specified in , then its default value - * (listed in table 4.attr) is used (it is said to be specified - * implicitly). If is NULL or empty (points to a list - * whose first value is zero), all attributes take on their default - * values. - * - * \param param_name may accept one of the following enumerated values: - * - CL_CURRENT_DEVICE_FOR_GL_CONTEXT_KHR 0x2006 - * - CL_DEVICES_FOR_GL_CONTEXT_KHR 0x2007. - * - * \param param_value_size is used to specify the size in bytes of memory - * pointed to by \a param_value. This size must be >= size of return type as - * described for \a param_name argumnet (GLenum or GLint). - * \a param_value_size_ret returns the actual size in bytes of data copied to - * \a param_value. If \a param_value_size_ret is NULL, it is ignored - * - * \param param_value is a pointer to memory where the appropriate result - * being queried is returned. If \a param_value is NULL, it is ignored. - * - * \param param_value_size is used to specify the size in bytes of memory - * pointed to by \a param_value. This size must be >= size of return type as - * described for \a param_name argumnet (GLenum or GLint). - * \a param_value_size_ret returns the actual size in bytes of data copied to - * \a param_value. If \a param_value_size_ret is NULL, it is ignored - * - * \return one of the following values is returned: - * - CL_SUCCESS if the function is executed successfully. - * - CL_SUCCESS if \a num_objects is 0 and \a mem_objects is NULL; the - * function does nothing. - * - CL_INVALID_VALUE if \a num_objects is zero and \a mem_objects is not a - * NULL value or if \a num_objects > 0 and \a mem_objects is NULL. - * - CL_INVALID_MEM_OBJECT if memory objects in \a mem_objects are not valid - * OpenCL memory objects. - * - CL_INVALID_COMMAND_QUEUE if \a command_queue is not a valid command-queue. - * - CL_INVALID_CONTEXT if clContext associated with \a command_queue was not - * created from an OpenGL clContext. - * - CL_INVALID_GL_OBJECT if memory objects in \a mem_objects have not been - * created from a GL object(s). - * - CL_INVALID_EVENT_WAIT_LIST if \a event_wait_list is NULL and - * \a num_events_in_wait_list > 0, or \a event_wait_list is not NULL and - * \a num_events_in_wait_list is 0, or if event objects in \a event_wait_list - * are not valid events. - * - CL_OUT_OF_HOST_MEMORY if there is a failure to allocate resources - * required by the OpenCL implementation on the host. - * - CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR if - * - * \version 1.0r47 - */ -RUNTIME_ENTRY(cl_int, clGetGLContextInfoKHR, - (const cl_context_properties* properties, cl_gl_context_info param_name, - size_t param_value_size, void* param_value, size_t* param_value_size_ret)) { - cl_int errcode=0; - cl_device_id* gpu_devices; - cl_uint num_gpu_devices = 0; - amd::Context::Info info; - static const bool VALIDATE_ONLY = true; - - errcode = amd::Context::checkProperties(properties, &info); - if (CL_SUCCESS != errcode) { - return errcode; - } - - if (!(info.flags_ & amd::Context::GLDeviceKhr)) { - // No GL context is specified - return CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR; - } - - // Get devices - //errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 0, NULL, &num_gpu_devices); - if (errcode != CL_SUCCESS && errcode != CL_DEVICE_NOT_FOUND) { - return CL_INVALID_VALUE; - } - - if (!num_gpu_devices) { - return CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR; - } - - switch (param_name) { - case CL_CURRENT_DEVICE_FOR_GL_CONTEXT_KHR: - // Return the CL device currently associated with the specified OpenGL context. - if (num_gpu_devices) { - gpu_devices = (cl_device_id*)alloca(num_gpu_devices * sizeof(cl_device_id)); - - //errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, num_gpu_devices, gpu_devices, NULL); - if (errcode != CL_SUCCESS) { - return errcode; - } - - for (cl_uint i = 0; i < num_gpu_devices; ++i) { - cl_device_id device = gpu_devices[i]; - if (is_valid(device) && - as_amd(device)->bindExternalDevice(info.flags_, info.hDev_, info.hCtx_, - VALIDATE_ONLY)) { - return amd::clGetInfo(device, param_value_size, param_value, param_value_size_ret); - } - } - - *not_null(param_value_size_ret) = 0; - } - break; - - case CL_DEVICES_FOR_GL_CONTEXT_KHR: { - // List of all CL devices that can be associated with the specified OpenGL context. - cl_uint total_devices = num_gpu_devices; - size_t size = total_devices * sizeof(cl_device_id); - - cl_device_id* devices = (cl_device_id*)alloca(size); - - //errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, total_devices, devices, NULL); - if (errcode != CL_SUCCESS) { - return errcode; - } - - std::vector compatible_devices; - - for (cl_uint i = 0; i < total_devices; ++i) { - cl_device_id device = devices[i]; - if (is_valid(device) && - as_amd(device)->bindExternalDevice(info.flags_, info.hDev_, info.hCtx_, - VALIDATE_ONLY)) { - compatible_devices.push_back(as_amd(device)); - } - } - - size_t deviceCount = compatible_devices.size(); - size_t deviceCountSize = deviceCount * sizeof(cl_device_id); - - if (param_value != NULL && param_value_size < deviceCountSize) { - return CL_INVALID_VALUE; - } - - *not_null(param_value_size_ret) = deviceCountSize; - - if (param_value != NULL) { - cl_device_id* deviceList = (cl_device_id*)param_value; - for (const auto& it : compatible_devices) { - *deviceList++ = as_cl(it); - } - } - - return CL_SUCCESS; - } break; - - default: - LogWarning("\"param_name\" is not valid"); - return CL_INVALID_VALUE; - } - return CL_SUCCESS; -} -RUNTIME_EXIT - -// -// -// namespace amd -// -// -namespace amd { - -typedef struct { - GLenum glBinding; - GLenum glTarget; -} TargetBindings_t; - -/*! @} - * \addtogroup CL-GL interop helper functions - * @{ - */ - -//! Function clearGLErrors() to clear all GL error bits, if any -void clearGLErrors(const Context& amdContext) { - GLenum glErr, glLastErr = GL_NO_ERROR; - while (1) { - glErr = amdContext.glenv()->glGetError_(); - if (glErr == GL_NO_ERROR || glErr == glLastErr) { - break; - } - glLastErr = glErr; - LogWarning("GL error"); - } -} - -GLenum checkForGLError(const Context& amdContext) { - GLenum glRetErr = GL_NO_ERROR; - GLenum glErr; - while (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - glRetErr = glErr; // Just return the last GL error - LogWarning("Check GL error"); - } - return glRetErr; -} - -//! Function getCLFormatFromGL returns "true" if GL format -//! is compatible with CL format, "false" otherwise. -bool getCLFormatFromGL(const Context& amdContext, GLint gliInternalFormat, - cl_image_format* pclImageFormat, int* piBytesPerPixel, cl_mem_flags flags) { - bool bRetVal = false; - - /* - Available values for "image_channel_order" - ========================================== - CL_R - CL_A - CL_INTENSITY - CL_LUMINANCE - CL_RG - CL_RA - CL_RGB - CL_RGBA - CL_ARGB - CL_BGRA - - Available values for "image_channel_data_type" - ============================================== - CL_SNORM_INT8 - CL_SNORM_INT16 - CL_UNORM_INT8 - CL_UNORM_INT16 - CL_UNORM_SHORT_565 - CL_UNORM_SHORT_555 - CL_UNORM_INT_101010 - CL_SIGNED_INT8 - CL_SIGNED_INT16 - CL_SIGNED_INT32 - CL_UNSIGNED_INT8 - CL_UNSIGNED_INT16 - CL_UNSIGNED_INT32 - CL_HALF_FLOAT - CL_FLOAT - */ - - switch (gliInternalFormat) { - case GL_RGB10_EXT: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = CL_UNORM_INT_101010; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_RGB10_A2: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = CL_UNORM_INT_101010; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_BGR8_ATI: - case GL_BGRA8_ATI: - pclImageFormat->image_channel_order = CL_BGRA; - pclImageFormat->image_channel_data_type = CL_UNORM_INT8; // CL_UNSIGNED_INT8; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_ALPHA8: - pclImageFormat->image_channel_order = CL_A; - pclImageFormat->image_channel_data_type = CL_UNORM_INT8; // CL_UNSIGNED_INT8; - *piBytesPerPixel = 1; - bRetVal = true; - break; - - case GL_R8: - case GL_R8UI: - pclImageFormat->image_channel_order = CL_R; - pclImageFormat->image_channel_data_type = - (gliInternalFormat == GL_R8) ? CL_UNORM_INT8 : CL_UNSIGNED_INT8; - *piBytesPerPixel = 1; - bRetVal = true; - break; - - case GL_R8I: - pclImageFormat->image_channel_order = CL_R; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT8; - *piBytesPerPixel = 1; - bRetVal = true; - break; - - case GL_RG8: - case GL_RG8UI: - pclImageFormat->image_channel_order = CL_RG; - pclImageFormat->image_channel_data_type = - (gliInternalFormat == GL_RG8) ? CL_UNORM_INT8 : CL_UNSIGNED_INT8; - *piBytesPerPixel = 2; - bRetVal = true; - break; - - case GL_RG8I: - pclImageFormat->image_channel_order = CL_RG; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT8; - *piBytesPerPixel = 2; - bRetVal = true; - break; - - case GL_RGB8: - case GL_RGB8UI: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = - (gliInternalFormat == GL_RGB8) ? CL_UNORM_INT8 : CL_UNSIGNED_INT8; - *piBytesPerPixel = 3; - bRetVal = true; - break; - - case GL_RGB8I: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT8; - *piBytesPerPixel = 3; - bRetVal = true; - break; - - case GL_RGBA: - case GL_RGBA8: - case GL_RGBA8UI: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = - (gliInternalFormat == GL_RGBA8UI) ? CL_UNSIGNED_INT8 : CL_UNORM_INT8; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_RGBA8I: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT8; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_R16: - case GL_R16UI: - pclImageFormat->image_channel_order = CL_R; - pclImageFormat->image_channel_data_type = - (gliInternalFormat == GL_R16) ? CL_UNORM_INT16 : CL_UNSIGNED_INT16; - bRetVal = true; - *piBytesPerPixel = 2; - break; - - case GL_R16I: - pclImageFormat->image_channel_order = CL_R; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT16; - *piBytesPerPixel = 2; - bRetVal = true; - break; - - case GL_R16F: - pclImageFormat->image_channel_order = CL_R; - pclImageFormat->image_channel_data_type = CL_HALF_FLOAT; - *piBytesPerPixel = 2; - bRetVal = true; - break; - - case GL_RG16: - case GL_RG16UI: - pclImageFormat->image_channel_order = CL_RG; - pclImageFormat->image_channel_data_type = - (gliInternalFormat == GL_RG16) ? CL_UNORM_INT16 : CL_UNSIGNED_INT16; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_RG16I: - pclImageFormat->image_channel_order = CL_RG; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT16; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_RG16F: - pclImageFormat->image_channel_order = CL_RG; - pclImageFormat->image_channel_data_type = CL_HALF_FLOAT; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_RGB16: - case GL_RGB16UI: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = - (gliInternalFormat == GL_RGB16) ? CL_UNORM_INT16 : CL_UNSIGNED_INT16; - *piBytesPerPixel = 6; - bRetVal = true; - break; - - case GL_RGB16I: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT16; - *piBytesPerPixel = 6; - bRetVal = true; - break; - - case GL_RGB16F: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = CL_HALF_FLOAT; - *piBytesPerPixel = 6; - bRetVal = true; - break; - - case GL_RGBA16: - case GL_RGBA16UI: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = - (gliInternalFormat == GL_RGBA16) ? CL_UNORM_INT16 : CL_UNSIGNED_INT16; - *piBytesPerPixel = 8; - bRetVal = true; - break; - - case GL_RGBA16I: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT16; - *piBytesPerPixel = 8; - bRetVal = true; - break; - - case GL_RGBA16F: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = CL_HALF_FLOAT; - *piBytesPerPixel = 8; - bRetVal = true; - break; - - case GL_R32I: - pclImageFormat->image_channel_order = CL_R; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT32; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_R32UI: - pclImageFormat->image_channel_order = CL_R; - pclImageFormat->image_channel_data_type = CL_UNSIGNED_INT32; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_R32F: - pclImageFormat->image_channel_order = CL_R; - pclImageFormat->image_channel_data_type = CL_FLOAT; - *piBytesPerPixel = 4; - bRetVal = true; - break; - - case GL_RG32I: - pclImageFormat->image_channel_order = CL_RG; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT32; - *piBytesPerPixel = 8; - bRetVal = true; - break; - - case GL_RG32UI: - pclImageFormat->image_channel_order = CL_RG; - pclImageFormat->image_channel_data_type = CL_UNSIGNED_INT32; - *piBytesPerPixel = 8; - bRetVal = true; - break; - - case GL_RG32F: - pclImageFormat->image_channel_order = CL_RG; - pclImageFormat->image_channel_data_type = CL_FLOAT; - *piBytesPerPixel = 8; - bRetVal = true; - break; - - case GL_RGB32I: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT32; - *piBytesPerPixel = 12; - bRetVal = true; - break; - - case GL_RGB32UI: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = CL_UNSIGNED_INT32; - *piBytesPerPixel = 12; - bRetVal = true; - break; - - case GL_RGB32F: - pclImageFormat->image_channel_order = CL_RGB; - pclImageFormat->image_channel_data_type = CL_FLOAT; - *piBytesPerPixel = 12; - bRetVal = true; - break; - - case GL_RGBA32I: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = CL_SIGNED_INT32; - *piBytesPerPixel = 16; - bRetVal = true; - break; - - case GL_RGBA32UI: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = CL_UNSIGNED_INT32; - *piBytesPerPixel = 16; - bRetVal = true; - break; - - case GL_RGBA32F: - pclImageFormat->image_channel_order = CL_RGBA; - pclImageFormat->image_channel_data_type = CL_FLOAT; - *piBytesPerPixel = 16; - bRetVal = true; - break; - case GL_DEPTH_COMPONENT32F: - pclImageFormat->image_channel_order = CL_DEPTH; - pclImageFormat->image_channel_data_type = CL_FLOAT; - *piBytesPerPixel = 4; - bRetVal = true; - break; - case GL_DEPTH_COMPONENT16: - pclImageFormat->image_channel_order = CL_DEPTH; - pclImageFormat->image_channel_data_type = CL_UNORM_INT16; - *piBytesPerPixel = 2; - bRetVal = true; - break; - case GL_DEPTH24_STENCIL8: - pclImageFormat->image_channel_order = CL_DEPTH_STENCIL; - pclImageFormat->image_channel_data_type = CL_UNORM_INT24; - *piBytesPerPixel = 4; - bRetVal = true; - break; - case GL_DEPTH32F_STENCIL8: - pclImageFormat->image_channel_order = CL_DEPTH_STENCIL; - pclImageFormat->image_channel_data_type = CL_FLOAT; - *piBytesPerPixel = 5; - bRetVal = true; - break; - default: - LogWarning("unsupported GL internal format"); - break; - } - amd::Image::Format imageFormat(*pclImageFormat); - if (bRetVal && !imageFormat.isSupported(amdContext, 0, flags)) { - bRetVal = false; - } - return bRetVal; -} - -void BufferGL::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(BufferGL)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -static GLenum clChannelDataTypeToGlType(cl_channel_type channel_type) { - // Pick - // GL_BYTE, GL_UNSIGNED_BYTE, GL_SHORT, GL_UNSIGNED_SHORT, GL_INT, - // GL_UNSIGNED_INT, GL_FLOAT, GL_2_BYTES, GL_3_BYTES, GL_4_BYTES - // or GL_DOUBLE - switch (channel_type) { - case CL_SNORM_INT8: - return GL_BYTE; - case CL_SNORM_INT16: - return GL_SHORT; - case CL_UNORM_INT8: - return GL_UNSIGNED_BYTE; - case CL_UNORM_INT16: - return GL_UNSIGNED_SHORT; - case CL_SIGNED_INT8: - return GL_BYTE; - case CL_SIGNED_INT16: - return GL_SHORT; - case CL_SIGNED_INT32: - return GL_INT; - case CL_UNSIGNED_INT8: - return GL_UNSIGNED_BYTE; - case CL_UNSIGNED_INT16: - return GL_UNSIGNED_SHORT; - case CL_UNSIGNED_INT32: - return GL_UNSIGNED_INT; - case CL_FLOAT: - return GL_FLOAT; - case CL_UNORM_INT_101010: - return GL_UNSIGNED_INT_10_10_10_2; - case CL_HALF_FLOAT: - case CL_UNORM_SHORT_565: - case CL_UNORM_SHORT_555: - default: - guarantee(false, "Unexpected CL type."); - return 0; - } -} - -static GLenum glInternalFormatToGlFormat(GLenum internalFormat) { - switch (internalFormat) { - // Base internal formats - case GL_RGBA: - case GL_BGRA: - return internalFormat; - // Sized internal formats - case GL_RGBA8: - case GL_RGBA16: - case GL_RGBA16F: - case GL_RGBA32F: - return GL_RGBA; - case GL_RGBA8I: - case GL_RGBA8UI: - case GL_RGBA16I: - case GL_RGBA16UI: - case GL_RGBA32I: - case GL_RGBA32UI: - return GL_RGBA_INTEGER; - - default: - guarantee(false, "Unexpected GL internal format."); - return 0; - } -} - -void ImageGL::initDeviceMemory() { - deviceMemories_ = - reinterpret_cast(reinterpret_cast(this) + sizeof(ImageGL)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); -} - -//******************************************************************* -// -// Internal implementation of CL API functions -// -//******************************************************************* - -// -// clCreateFromGLBufferAMD -// -cl_mem clCreateFromGLBufferAMD(Context& amdContext, cl_mem_flags flags, GLuint bufobj, - cl_int* errcode_ret) { - BufferGL* pBufferGL = NULL; - GLenum glErr; - GLenum glTarget = GL_ARRAY_BUFFER; - GLint gliSize = 0; - GLint gliMapped = 0; - - // Verify context init'ed for interop - if (!amdContext.glenv() || !amdContext.glenv()->isAssociated()) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("\"amdContext\" is not created from GL context or share list"); - return (cl_mem)0; - } - - // Add this scope to bound the scoped lock - { - GLFunctions::SetIntEnv ie(amdContext.glenv()); - if (!ie.isValid()) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("\"amdContext\" is not created from GL context or share list"); - return as_cl(0); - } - - // Verify GL buffer object - clearGLErrors(amdContext); - if ((GL_FALSE == amdContext.glenv()->glIsBuffer_(bufobj)) || - (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_()))) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("\"bufobj\" is not a GL buffer object"); - return (cl_mem)0; - } - - // It seems that CL spec is not concerned with GL_BUFFER_USAGE, so skip it - - // Check if size is available - data store is created - - amdContext.glenv()->glBindBuffer_(glTarget, bufobj); - clearGLErrors(amdContext); - amdContext.glenv()->glGetBufferParameteriv_(glTarget, GL_BUFFER_SIZE, &gliSize); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("cannot get the GL buffer size"); - return (cl_mem)0; - } - if (gliSize == 0) { - //@todo - check why sometime the size is zero - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("the GL buffer's data store is not created"); - return (cl_mem)0; - } - - // Mapping will be done at acquire time (sync point) - - } // Release scoped lock - - // Now create BufferGL object - pBufferGL = new (amdContext) BufferGL(amdContext, flags, gliSize, 0, bufobj); - - if (!pBufferGL) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - LogWarning("cannot create object of class BufferGL"); - return (cl_mem)0; - } - - if (!pBufferGL->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pBufferGL->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - - // Create interop object - if (pBufferGL->getInteropObj() == NULL) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("cannot create object of class BufferGL"); - return (cl_mem)0; - } - - // Fixme: If more than one device is present in the context, we choose the first device. - // We should come up with a more elegant solution to handle this. - assert(amdContext.devices().size() == 1); - - const auto it = amdContext.devices().cbegin(); - const amd::Device& dev = *(*it); - - device::Memory* mem = pBufferGL->getDeviceMemory(dev); - if (NULL == mem) { - LogPrintfError("Can't allocate memory size - 0x%08X bytes!", pBufferGL->getSize()); - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - return (cl_mem)0; - } - mem->processGLResource(device::Memory::GLDecompressResource); - - return as_cl(pBufferGL); -} - -cl_mem clCreateFromGLTextureAMD(Context& amdContext, cl_mem_flags clFlags, GLenum target, - GLint miplevel, GLuint texture, int* errcode_ret) { - ImageGL* pImageGL = NULL; - GLenum glErr; - GLenum glTarget = 0; - GLenum glInternalFormat; - cl_image_format clImageFormat; - uint dim = 1; - cl_mem_object_type clType; - cl_gl_object_type clGLType; - GLsizei numSamples = 1; - - // Verify context init'ed for interop - if (!amdContext.glenv() || !amdContext.glenv()->isAssociated()) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("\"amdContext\" is not created from GL context or share list"); - return static_cast(0); - } - - GLint gliTexWidth = 1; - GLint gliTexHeight = 1; - GLint gliTexDepth = 1; - - // Add this scope to bound the scoped lock - { - GLFunctions::SetIntEnv ie(amdContext.glenv()); - if (!ie.isValid()) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("\"amdContext\" is not created from GL context or share list"); - return as_cl(0); - } - - // Verify GL texture object - clearGLErrors(amdContext); - if ((GL_FALSE == amdContext.glenv()->glIsTexture_(texture)) || - (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_()))) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("\"texture\" is not a GL texture object"); - return static_cast(0); - } - - bool image = true; - - // Check target value validity - switch (target) { - case GL_TEXTURE_BUFFER: - glTarget = GL_TEXTURE_BUFFER; - dim = 1; - clType = CL_MEM_OBJECT_IMAGE1D_BUFFER; - clGLType = CL_GL_OBJECT_TEXTURE_BUFFER; - image = false; - break; - - case GL_TEXTURE_1D: - glTarget = GL_TEXTURE_1D; - dim = 1; - clType = CL_MEM_OBJECT_IMAGE1D; - clGLType = CL_GL_OBJECT_TEXTURE1D; - break; - - case GL_TEXTURE_CUBE_MAP_POSITIVE_X: - case GL_TEXTURE_CUBE_MAP_NEGATIVE_X: - case GL_TEXTURE_CUBE_MAP_POSITIVE_Y: - case GL_TEXTURE_CUBE_MAP_NEGATIVE_Y: - case GL_TEXTURE_CUBE_MAP_POSITIVE_Z: - case GL_TEXTURE_CUBE_MAP_NEGATIVE_Z: - glTarget = GL_TEXTURE_CUBE_MAP; - dim = 2; - clType = CL_MEM_OBJECT_IMAGE2D; - clGLType = CL_GL_OBJECT_TEXTURE2D; - break; - - case GL_TEXTURE_1D_ARRAY: - glTarget = GL_TEXTURE_1D_ARRAY; - dim = 2; - clType = CL_MEM_OBJECT_IMAGE1D_ARRAY; - clGLType = CL_GL_OBJECT_TEXTURE1D_ARRAY; - break; - - case GL_TEXTURE_2D: - glTarget = GL_TEXTURE_2D; - dim = 2; - clType = CL_MEM_OBJECT_IMAGE2D; - clGLType = CL_GL_OBJECT_TEXTURE2D; - break; - - case GL_TEXTURE_2D_MULTISAMPLE: - glTarget = GL_TEXTURE_2D_MULTISAMPLE; - dim = 2; - clType = CL_MEM_OBJECT_IMAGE2D; - clGLType = CL_GL_OBJECT_TEXTURE2D; - break; - - case GL_TEXTURE_RECTANGLE_ARB: - glTarget = GL_TEXTURE_RECTANGLE_ARB; - dim = 2; - clType = CL_MEM_OBJECT_IMAGE2D; - clGLType = CL_GL_OBJECT_TEXTURE2D; - break; - - case GL_TEXTURE_2D_ARRAY: - glTarget = GL_TEXTURE_2D_ARRAY; - dim = 3; - clType = CL_MEM_OBJECT_IMAGE2D_ARRAY; - clGLType = CL_GL_OBJECT_TEXTURE2D_ARRAY; - break; - - case GL_TEXTURE_3D: - glTarget = GL_TEXTURE_3D; - dim = 3; - clType = CL_MEM_OBJECT_IMAGE3D; - clGLType = CL_GL_OBJECT_TEXTURE3D; - break; - - default: - // wrong value - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid \"target\" value"); - return static_cast(0); - break; - } - - amdContext.glenv()->glBindTexture_(glTarget, texture); - - // Check if size is available - data store is created - if (image) { - // Check mipmap level for "texture" name - GLint gliTexBaseLevel; - GLint gliTexMaxLevel; - - clearGLErrors(amdContext); - amdContext.glenv()->glGetTexParameteriv_(glTarget, GL_TEXTURE_BASE_LEVEL, &gliTexBaseLevel); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_MIP_LEVEL; - LogWarning("Cannot get base mipmap level of a GL \"texture\" object"); - return static_cast(0); - } - clearGLErrors(amdContext); - amdContext.glenv()->glGetTexParameteriv_(glTarget, GL_TEXTURE_MAX_LEVEL, &gliTexMaxLevel); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_MIP_LEVEL; - LogWarning("Cannot get max mipmap level of a GL \"texture\" object"); - return static_cast(0); - } - if ((gliTexBaseLevel > miplevel) || (miplevel > gliTexMaxLevel)) { - *not_null(errcode_ret) = CL_INVALID_MIP_LEVEL; - LogWarning("\"miplevel\" is not a valid mipmap level of the GL \"texture\" object"); - return static_cast(0); - } - - // Get GL texture format and check if it's compatible with CL format - clearGLErrors(amdContext); - amdContext.glenv()->glGetTexLevelParameteriv_(target, miplevel, GL_TEXTURE_INTERNAL_FORMAT, - (GLint*)&glInternalFormat); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("Cannot get internal format of \"miplevel\" of GL \"texture\" object"); - return static_cast(0); - } - - amdContext.glenv()->glGetTexLevelParameteriv_(target, miplevel, GL_TEXTURE_SAMPLES, - (GLint*)&numSamples); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("Cannot get numbers of samples of GL \"texture\" object"); - return static_cast(0); - } - if (numSamples > 1) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("MSAA \"texture\" object is not suppoerted for the device"); - return static_cast(0); - } - - // Now get CL format from GL format and bytes per pixel - int iBytesPerPixel = 0; - if (!getCLFormatFromGL(amdContext, glInternalFormat, &clImageFormat, &iBytesPerPixel, - clFlags)) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("\"texture\" format does not map to an appropriate CL image format"); - return static_cast(0); - } - - switch (dim) { - case 3: - clearGLErrors(amdContext); - amdContext.glenv()->glGetTexLevelParameteriv_(target, miplevel, GL_TEXTURE_DEPTH, - &gliTexDepth); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("Cannot get the depth of \"miplevel\" of GL \"texure\""); - return static_cast(0); - } - // Fall trough to process other dimensions... - case 2: - clearGLErrors(amdContext); - amdContext.glenv()->glGetTexLevelParameteriv_(target, miplevel, GL_TEXTURE_HEIGHT, - &gliTexHeight); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("Cannot get the height of \"miplevel\" of GL \"texure\""); - return static_cast(0); - } - // Fall trough to process other dimensions... - case 1: - clearGLErrors(amdContext); - amdContext.glenv()->glGetTexLevelParameteriv_(target, miplevel, GL_TEXTURE_WIDTH, - &gliTexWidth); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("Cannot get the width of \"miplevel\" of GL \"texure\""); - return static_cast(0); - } - break; - default: - *not_null(errcode_ret) = CL_INVALID_VALUE; - LogWarning("invalid \"target\" value"); - return static_cast(0); - } - } else { - GLint size; - - // In case target is GL_TEXTURE_BUFFER - GLint backingBuffer; - clearGLErrors(amdContext); - amdContext.glenv()->glGetTexLevelParameteriv_( - glTarget, 0, GL_TEXTURE_BUFFER_DATA_STORE_BINDING, &backingBuffer); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("Cannot get backing buffer for GL \"texture buffer\" object"); - return static_cast(0); - } - amdContext.glenv()->glBindBuffer_(glTarget, backingBuffer); - - // Get GL texture format and check if it's compatible with CL format - clearGLErrors(amdContext); - amdContext.glenv()->glGetIntegerv_(GL_TEXTURE_BUFFER_FORMAT_EXT, - reinterpret_cast(&glInternalFormat)); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("Cannot get internal format of \"miplevel\" of GL \"texture\" object"); - return static_cast(0); - } - - // Now get CL format from GL format and bytes per pixel - int iBytesPerPixel = 0; - if (!getCLFormatFromGL(amdContext, glInternalFormat, &clImageFormat, &iBytesPerPixel, - clFlags)) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("\"texture\" format does not map to an appropriate CL image format"); - return static_cast(0); - } - - clearGLErrors(amdContext); - amdContext.glenv()->glGetBufferParameteriv_(glTarget, GL_BUFFER_SIZE, &size); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("Cannot get internal format of \"miplevel\" of GL \"texture\" object"); - return static_cast(0); - } - - gliTexWidth = size / iBytesPerPixel; - } - size_t imageSize = (clType == CL_MEM_OBJECT_IMAGE1D_ARRAY) ? static_cast(gliTexHeight) - : static_cast(gliTexDepth); - - if (!amd::Image::validateDimensions( - amdContext.devices(), clType, static_cast(gliTexWidth), - static_cast(gliTexHeight), static_cast(gliTexDepth), imageSize)) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("The GL \"texture\" data store is not created or out of supported dimensions"); - return static_cast(0); - } - - // PBO and mapping will be done at "acquire" time (sync point) - - } // Release scoped lock - - target = (glTarget == GL_TEXTURE_CUBE_MAP) ? target : 0; - - pImageGL = new (amdContext) - ImageGL(amdContext, clType, clFlags, clImageFormat, static_cast(gliTexWidth), - static_cast(gliTexHeight), static_cast(gliTexDepth), glTarget, - texture, miplevel, glInternalFormat, clGLType, numSamples, target); - - if (!pImageGL) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - LogWarning("Cannot create class ImageGL - out of memory?"); - return static_cast(0); - } - - if (!pImageGL->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImageGL->release(); - return static_cast(0); - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImageGL); -} - -// -// clCreateFromGLRenderbufferDAMD -// -cl_mem clCreateFromGLRenderbufferAMD(Context& amdContext, cl_mem_flags clFlags, GLuint renderbuffer, - int* errcode_ret) { - ImageGL* pImageGL = NULL; - GLenum glErr; - - GLenum glTarget = GL_RENDERBUFFER; - GLenum glInternalFormat; - cl_image_format clImageFormat; - - // Verify context init'ed for interop - if (!amdContext.glenv() || !amdContext.glenv()->isAssociated()) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("\"amdContext\" is not created from GL context or share list"); - return (cl_mem)0; - } - - GLint gliRbWidth; - GLint gliRbHeight; - - // Add this scope to bound the scoped lock - { - GLFunctions::SetIntEnv ie(amdContext.glenv()); - if (!ie.isValid()) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("\"amdContext\" is not created from GL context or share list"); - return as_cl(0); - } - - // Verify GL renderbuffer object - clearGLErrors(amdContext); - if ((GL_FALSE == amdContext.glenv()->glIsRenderbufferEXT_(renderbuffer)) || - (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_()))) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("\"renderbuffer\" is not a GL texture object"); - return (cl_mem)0; - } - - amdContext.glenv()->glBindRenderbuffer_(glTarget, renderbuffer); - - // Get GL RB format and check if it's compatible with CL format - clearGLErrors(amdContext); - amdContext.glenv()->glGetRenderbufferParameterivEXT_(glTarget, GL_RENDERBUFFER_INTERNAL_FORMAT, - (GLint*)&glInternalFormat); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("Cannot get internal format of GL \"renderbuffer\" object"); - return (cl_mem)0; - } - - // Now get CL format from GL format and bytes per pixel - int iBytesPerPixel = 0; - if (!getCLFormatFromGL(amdContext, glInternalFormat, &clImageFormat, &iBytesPerPixel, - clFlags)) { - *not_null(errcode_ret) = CL_INVALID_IMAGE_FORMAT_DESCRIPTOR; - LogWarning("\"renderbuffer\" format does not map to an appropriate CL image format"); - return (cl_mem)0; - } - - // Check if size is available - data store is created - clearGLErrors(amdContext); - amdContext.glenv()->glGetRenderbufferParameterivEXT_(glTarget, GL_RENDERBUFFER_WIDTH, - &gliRbWidth); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("Cannot get the width of GL \"renderbuffer\""); - return (cl_mem)0; - } - if (gliRbWidth == 0) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("The GL \"renderbuffer\" data store is not created"); - return (cl_mem)0; - } - clearGLErrors(amdContext); - amdContext.glenv()->glGetRenderbufferParameterivEXT_(glTarget, GL_RENDERBUFFER_HEIGHT, - &gliRbHeight); - if (GL_NO_ERROR != (glErr = amdContext.glenv()->glGetError_())) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("Cannot get the height of GL \"renderbuffer\""); - return (cl_mem)0; - } - if (gliRbHeight == 0) { - *not_null(errcode_ret) = CL_INVALID_GL_OBJECT; - LogWarning("The GL \"renderbuffer\" data store is not created"); - return (cl_mem)0; - } - - // PBO and mapping will be done at "acquire" time (sync point) - - } // Release scoped lock - - pImageGL = - new (amdContext) ImageGL(amdContext, CL_MEM_OBJECT_IMAGE2D, clFlags, clImageFormat, - (size_t)gliRbWidth, (size_t)gliRbHeight, 1, glTarget, renderbuffer, - 0, glInternalFormat, CL_GL_OBJECT_RENDERBUFFER, 0); - - if (!pImageGL) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - LogWarning("Cannot create class ImageGL from renderbuffer - out of memory?"); - return (cl_mem)0; - } - - if (!pImageGL->create()) { - *not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE; - pImageGL->release(); - return (cl_mem)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(pImageGL); -} - -// -// clEnqueueAcquireExtObjectsAMD -// - -static cl_int clSetInteropObjects(cl_uint num_objects, const cl_mem* mem_objects, - std::vector& interopObjects) { - if ((num_objects == 0 && mem_objects != NULL) || (num_objects != 0 && mem_objects == NULL)) { - return CL_INVALID_VALUE; - } - - while (num_objects-- > 0) { - cl_mem obj = *mem_objects++; - if (!is_valid(obj)) { - return CL_INVALID_MEM_OBJECT; - } - - amd::Memory* mem = as_amd(obj); - if (mem->getInteropObj() == NULL) { - return CL_INVALID_GL_OBJECT; - } - - interopObjects.push_back(mem); - } - return CL_SUCCESS; -} - -cl_int clEnqueueAcquireExtObjectsAMD(cl_command_queue command_queue, cl_uint num_objects, - const cl_mem* mem_objects, cl_uint num_events_in_wait_list, - const cl_event* event_wait_list, cl_event* event, - cl_command_type cmd_type) { - if (!is_valid(command_queue)) { - return CL_INVALID_COMMAND_QUEUE; - } - - amd::HostQueue* queue = as_amd(command_queue)->asHostQueue(); - if (NULL == queue) { - return CL_INVALID_COMMAND_QUEUE; - } - amd::HostQueue& hostQueue = *queue; - - if (cmd_type == CL_COMMAND_ACQUIRE_GL_OBJECTS) { - // Verify context init'ed for interop - if (!hostQueue.context().glenv() || !hostQueue.context().glenv()->isAssociated()) { - LogWarning("\"amdContext\" is not created from GL context or share list"); - return CL_INVALID_CONTEXT; - } - } - - std::vector memObjects; - cl_int err = clSetInteropObjects(num_objects, mem_objects, memObjects); - if (err != CL_SUCCESS) { - return err; - } - - amd::Command::EventWaitList eventWaitList; - err = amd::clSetEventWaitList(eventWaitList, hostQueue, num_events_in_wait_list, - event_wait_list); - if (err != CL_SUCCESS) { - return err; - } - -#ifdef _WIN32 - if ((hostQueue.context().info().flags_ & amd::Context::InteropUserSync) == 0) { - //! Make sure D3D10 queues are flushed and all commands are finished - //! before CL side would access interop objects - if (cmd_type == CL_COMMAND_ACQUIRE_D3D10_OBJECTS_KHR) { - SyncD3D10Objects(memObjects); - } - //! Make sure D3D11 queues are flushed and all commands are finished - //! before CL side would access interop objects - if (cmd_type == CL_COMMAND_ACQUIRE_D3D11_OBJECTS_KHR) { - SyncD3D11Objects(memObjects); - } - //! Make sure D3D9 queues are flushed and all commands are finished - //! before CL side would access interop objects - if (cmd_type == CL_COMMAND_ACQUIRE_DX9_MEDIA_SURFACES_KHR) { - SyncD3D9Objects(memObjects); - } - } -#endif //_WIN32 - - //! Now create command and enqueue - amd::AcquireExtObjectsCommand* command = new amd::AcquireExtObjectsCommand( - hostQueue, eventWaitList, num_objects, memObjects, cmd_type); - if (command == NULL) { - return CL_OUT_OF_HOST_MEMORY; - } - - // Make sure we have memory for the command execution - if (!command->validateMemory()) { - delete command; - return CL_MEM_OBJECT_ALLOCATION_FAILURE; - } - - command->enqueue(); - - *not_null(event) = as_cl(&command->event()); - if (event == NULL) { - command->release(); - } - return CL_SUCCESS; -} - - -// -// clEnqueueReleaseExtObjectsAMD -// -cl_int clEnqueueReleaseExtObjectsAMD(cl_command_queue command_queue, cl_uint num_objects, - const cl_mem* mem_objects, cl_uint num_events_in_wait_list, - const cl_event* event_wait_list, cl_event* event, - cl_command_type cmd_type) { - if (!is_valid(command_queue)) { - return CL_INVALID_COMMAND_QUEUE; - } - - amd::HostQueue* queue = as_amd(command_queue)->asHostQueue(); - if (NULL == queue) { - return CL_INVALID_COMMAND_QUEUE; - } - amd::HostQueue& hostQueue = *queue; - - std::vector memObjects; - cl_int err = clSetInteropObjects(num_objects, mem_objects, memObjects); - if (err != CL_SUCCESS) { - return err; - } - - amd::Command::EventWaitList eventWaitList; - err = amd::clSetEventWaitList(eventWaitList, hostQueue, num_events_in_wait_list, - event_wait_list); - if (err != CL_SUCCESS) { - return err; - } - - //! Now create command and enqueue - amd::ReleaseExtObjectsCommand* command = new amd::ReleaseExtObjectsCommand( - hostQueue, eventWaitList, num_objects, memObjects, cmd_type); - if (command == NULL) { - return CL_OUT_OF_HOST_MEMORY; - } - - // Make sure we have memory for the command execution - if (!command->validateMemory()) { - delete command; - return CL_MEM_OBJECT_ALLOCATION_FAILURE; - } - - command->enqueue(); - -#ifdef _WIN32 - if ((hostQueue.context().info().flags_ & amd::Context::InteropUserSync) == 0) { - //! Make sure CL command queue is flushed and all commands are finished - //! before D3D10 side would access interop resources - if (cmd_type == CL_COMMAND_RELEASE_DX9_MEDIA_SURFACES_KHR || - cmd_type == CL_COMMAND_RELEASE_D3D10_OBJECTS_KHR || - cmd_type == CL_COMMAND_RELEASE_D3D11_OBJECTS_KHR) { - command->awaitCompletion(); - } - } -#endif //_WIN32 - - *not_null(event) = as_cl(&command->event()); - - if (event == NULL) { - command->release(); - } - - return CL_SUCCESS; -} - -// Placed here as opposed to command.cpp, as glext.h and cl_gl_amd.hpp will have -// to be included because of the GL calls -bool ClGlEvent::waitForFence() { - GLenum ret; - // get fence id associated with fence event - GLsync gs = reinterpret_cast(command().data()); - if (!gs) return false; - -// Try to use DC and GLRC of current thread, if it doesn't exist -// create a new GL context on this thread, which is shared with the original context - -#ifdef _WIN32 - HDC tempDC_ = wglGetCurrentDC(); - HGLRC tempGLRC_ = wglGetCurrentContext(); - // Set DC and GLRC - if (tempDC_ && tempGLRC_) { - ret = context().glenv()->glClientWaitSync_(gs, GL_SYNC_FLUSH_COMMANDS_BIT, - static_cast(-1)); - if (!(ret == GL_ALREADY_SIGNALED || ret == GL_CONDITION_SATISFIED)) return false; - } else { - tempDC_ = context().glenv()->getDC(); - tempGLRC_ = context().glenv()->getIntGLRC(); - if (!context().glenv()->init(reinterpret_cast(tempDC_), - reinterpret_cast(tempGLRC_))) - return false; - - // Make the newly created GL context current to this thread - context().glenv()->setIntEnv(); - // If fence has not yet executed, wait till it finishes - ret = context().glenv()->glClientWaitSync_(gs, GL_SYNC_FLUSH_COMMANDS_BIT, - static_cast(-1)); - if (!(ret == GL_ALREADY_SIGNALED || ret == GL_CONDITION_SATISFIED)) return false; - // Since we're done making GL calls, restore whatever context was previously current to this - // thread - context().glenv()->restoreEnv(); - } -#else // Lnx - Display* tempDpy_ = context().glenv()->glXGetCurrentDisplay_(); - GLXDrawable tempDrawable_ = context().glenv()->glXGetCurrentDrawable_(); - GLXContext tempCtx_ = context().glenv()->glXGetCurrentContext_(); - // Set internal Display and GLXContext - if (tempDpy_ && tempCtx_) { - ret = context().glenv()->glClientWaitSync_(gs, GL_SYNC_FLUSH_COMMANDS_BIT, - static_cast(-1)); - if (!(ret == GL_ALREADY_SIGNALED || ret == GL_CONDITION_SATISFIED)) return false; - } else { - if (!context().glenv()->init(reinterpret_cast(context().glenv()->getIntDpy()), - reinterpret_cast(context().glenv()->getIntCtx()))) - return false; - - // Make the newly created GL context current to this thread - context().glenv()->setIntEnv(); - // If fence has not yet executed, wait till it finishes - ret = context().glenv()->glClientWaitSync_(gs, GL_SYNC_FLUSH_COMMANDS_BIT, - static_cast(-1)); - if (!(ret == GL_ALREADY_SIGNALED || ret == GL_CONDITION_SATISFIED)) return false; - // Since we're done making GL calls, restore whatever context was previously current to this - // thread - context().glenv()->restoreEnv(); - } -#endif - // If we reach this point, fence should have completed - setStatus(CL_COMPLETE); - return true; -} - -// -// GLFunctions implementation -// - -#ifdef _WIN32 -#define CONVERT_CHAR_GLUBYTE -#else //!_WIN32 -#define CONVERT_CHAR_GLUBYTE (GLubyte*) -#endif //!_WIN32 - -#define GLPREFIX(rtype, fcn, dclargs) \ - if (!(fcn##_ = (PFN_##fcn)GETPROCADDRESS(libHandle_, #fcn))) { \ - if (!(fcn##_ = (PFN_##fcn)GetProcAddress_(reinterpret_cast(#fcn)))) ++missed_; \ - } - -GLFunctions::SetIntEnv::SetIntEnv(GLFunctions* env) : env_(env) { - env_->getLock().lock(); - - // Set environment (DC and GLRC) - isValid_ = env_->setIntEnv(); -} - -GLFunctions::SetIntEnv::~SetIntEnv() { - // Restore environment (CL DC and CL GLRC) - env_->restoreEnv(); - - env_->getLock().unlock(); -} - -GLFunctions::GLFunctions(HMODULE h, bool isEGL) - : libHandle_(h), - missed_(0), - eglDisplay_(EGL_NO_DISPLAY), - eglOriginalContext_(EGL_NO_CONTEXT), - eglInternalContext_(EGL_NO_CONTEXT), - eglTempContext_(EGL_NO_CONTEXT), - isEGL_(isEGL), -#ifdef _WIN32 - hOrigGLRC_(0), - hDC_(0), - hIntGLRC_(0) -#else //!_WIN32 - Dpy_(0), - Drawable_(0), - origCtx_(0), - intDpy_(0), - intDrawable_(0), - intCtx_(0), - XOpenDisplay_(NULL), - XCloseDisplay_(NULL), - glXGetCurrentDrawable_(NULL), - glXGetCurrentDisplay_(NULL), - glXGetCurrentContext_(NULL), - glXChooseVisual_(NULL), - glXCreateContext_(NULL), - glXDestroyContext_(NULL), - glXMakeCurrent_(NULL) -#endif //!_WIN32 -{ -#define VERIFY_POINTER(p) \ - if (NULL == p) { \ - missed_++; \ - } - - if (isEGL_) { - GetProcAddress_ = (PFN_xxxGetProcAddress)GETPROCADDRESS(h, "eglGetProcAddress"); - } else { - GetProcAddress_ = (PFN_xxxGetProcAddress)GETPROCADDRESS(h, API_GETPROCADDR); - } -#ifndef _WIN32 - // Initialize pointers to X11/GLX functions - // We can not link with these functions on compile time since we need to support - // console mode. In console mode X server and X server components may be absent. - // Hence linking with X11 or libGL will fail module image loading in console mode.-tzachi cohen - - if (!isEGL_) { - glXGetCurrentDrawable_ = (PFNglXGetCurrentDrawable)GETPROCADDRESS(h, "glXGetCurrentDrawable"); - VERIFY_POINTER(glXGetCurrentDrawable_) - glXGetCurrentDisplay_ = (PFNglXGetCurrentDisplay)GETPROCADDRESS(h, "glXGetCurrentDisplay"); - VERIFY_POINTER(glXGetCurrentDisplay_) - glXGetCurrentContext_ = (PFNglXGetCurrentContext)GETPROCADDRESS(h, "glXGetCurrentContext"); - VERIFY_POINTER(glXGetCurrentContext_) - glXChooseVisual_ = (PFNglXChooseVisual)GETPROCADDRESS(h, "glXChooseVisual"); - VERIFY_POINTER(glXChooseVisual_) - glXCreateContext_ = (PFNglXCreateContext)GETPROCADDRESS(h, "glXCreateContext"); - VERIFY_POINTER(glXCreateContext_) - glXDestroyContext_ = (PFNglXDestroyContext)GETPROCADDRESS(h, "glXDestroyContext"); - VERIFY_POINTER(glXDestroyContext_) - glXMakeCurrent_ = (PFNglXMakeCurrent)GETPROCADDRESS(h, "glXMakeCurrent"); - VERIFY_POINTER(glXMakeCurrent_) - - HMODULE hXModule = (HMODULE)Os::loadLibrary("libX11.so.6"); - if (NULL != hXModule) { - XOpenDisplay_ = (PFNXOpenDisplay)GETPROCADDRESS(hXModule, "XOpenDisplay"); - VERIFY_POINTER(XOpenDisplay_) - XCloseDisplay_ = (PFNXCloseDisplay)GETPROCADDRESS(hXModule, "XCloseDisplay"); - VERIFY_POINTER(XCloseDisplay_) - } else { - missed_ += 2; - } - } -// Initialize pointers to GL functions -#include "gl_functions.hpp" -#else - if (!isEGL_) { - wglCreateContext_ = (PFN_wglCreateContext)GETPROCADDRESS(h, "wglCreateContext"); - VERIFY_POINTER(wglCreateContext_) - wglGetCurrentContext_ = (PFN_wglGetCurrentContext)GETPROCADDRESS(h, "wglGetCurrentContext"); - VERIFY_POINTER(wglGetCurrentContext_) - wglGetCurrentDC_ = (PFN_wglGetCurrentDC)GETPROCADDRESS(h, "wglGetCurrentDC"); - VERIFY_POINTER(wglGetCurrentDC_) - wglDeleteContext_ = (PFN_wglDeleteContext)GETPROCADDRESS(h, "wglDeleteContext"); - VERIFY_POINTER(wglDeleteContext_) - wglMakeCurrent_ = (PFN_wglMakeCurrent)GETPROCADDRESS(h, "wglMakeCurrent"); - VERIFY_POINTER(wglMakeCurrent_) - wglShareLists_ = (PFN_wglShareLists)GETPROCADDRESS(h, "wglShareLists"); - VERIFY_POINTER(wglShareLists_) - } -#endif -} - -GLFunctions::~GLFunctions() { -#ifdef _WIN32 - if (hIntGLRC_) { - if (!wglDeleteContext_(hIntGLRC_)) { - DWORD dwErr = GetLastError(); - LogWarning("Cannot delete GLRC"); - } - } -#else //!_WIN32 - if (intDpy_) { - if (intCtx_) { - glXDestroyContext_(intDpy_, intCtx_); - intCtx_ = NULL; - } - XCloseDisplay_(intDpy_); - intDpy_ = NULL; - } -#endif //!_WIN32 -} - -bool GLFunctions::init(intptr_t hdc, intptr_t hglrc) { - if (isEGL_) { - eglDisplay_ = (EGLDisplay)hdc; - eglOriginalContext_ = (EGLContext)hglrc; - return true; - } - -#ifdef _WIN32 - DWORD err; - - if (missed_) { - return false; - } - - if (!hdc) { - hDC_ = wglGetCurrentDC_(); - } else { - hDC_ = (HDC)hdc; - } - hOrigGLRC_ = (HGLRC)hglrc; - if (!(hIntGLRC_ = wglCreateContext_(hDC_))) { - err = GetLastError(); - return false; - } - if (!wglShareLists_(hOrigGLRC_, hIntGLRC_)) { - err = GetLastError(); - return false; - } - - bool makeCurrentNull = false; - - if (wglGetCurrentContext_() == NULL) { - wglMakeCurrent_(hDC_, hIntGLRC_); - - makeCurrentNull = true; - } - -// Initialize pointers to GL functions -#include "gl_functions.hpp" - - if (makeCurrentNull) { - wglMakeCurrent_(NULL, NULL); - } - - if (missed_ == 0) { - return true; - } -#else //!_WIN32 - if (!missed_) { - if (!hdc) { - Dpy_ = glXGetCurrentDisplay_(); - } else { - Dpy_ = (Display*)hdc; - } - Drawable_ = glXGetCurrentDrawable_(); - origCtx_ = (GLXContext)hglrc; - - int attribList[] = {GLX_RGBA, None}; - if (!(intDpy_ = XOpenDisplay_(DisplayString(Dpy_)))) { -#if defined(ATI_ARCH_X86) - asm("int $3"); -#endif - } - intDrawable_ = DefaultRootWindow(intDpy_); - - XVisualInfo* vis; - int defaultScreen = DefaultScreen(intDpy_); - if (!(vis = glXChooseVisual_(intDpy_, defaultScreen, attribList))) { - return false; - } - if (!(intCtx_ = glXCreateContext_(intDpy_, vis, origCtx_, true))) { - return false; - } - return true; - } -#endif //!_WIN32 - return false; -} - -bool GLFunctions::setIntEnv() { - if (isEGL_) { - return true; - } -#ifdef _WIN32 - // Save current DC and GLRC - tempDC_ = wglGetCurrentDC_(); - tempGLRC_ = wglGetCurrentContext_(); - // Set internal DC and GLRC - if (tempDC_ != getDC() || tempGLRC_ != getIntGLRC()) { - if (!wglMakeCurrent_(getDC(), getIntGLRC())) { - DWORD err = GetLastError(); - LogWarning("cannot set internal GL environment"); - return false; - } - } -#else //!_WIN32 - tempDpy_ = glXGetCurrentDisplay_(); - tempDrawable_ = glXGetCurrentDrawable_(); - tempCtx_ = glXGetCurrentContext_(); - // Set internal Display and GLXContext - if (tempDpy_ != getDpy() || tempCtx_ != getIntCtx()) { - if (!glXMakeCurrent_(getIntDpy(), getIntDrawable(), getIntCtx())) { - LogWarning("cannot set internal GL environment"); - return false; - } - } -#endif //!_WIN32 - - return true; -} - -bool GLFunctions::restoreEnv() { - if (isEGL_) { - // eglMakeCurrent( ); - return true; - } -#ifdef _WIN32 - // Restore original DC and GLRC - if (!wglMakeCurrent_(tempDC_, tempGLRC_)) { - DWORD err = GetLastError(); - LogWarning("cannot restore original GL environment"); - return false; - } -#else //!_WIN32 - // Restore Display and GLXContext - if (tempDpy_) { - if (!glXMakeCurrent_(tempDpy_, tempDrawable_, tempCtx_)) { - LogWarning("cannot restore original GL environment"); - return false; - } - } else { - // Just release internal context - if (!glXMakeCurrent_(getIntDpy(), None, NULL)) { - LogWarning("cannot reelase internal GL environment"); - return false; - } - } -#endif //!_WIN32 - - return true; -} - -} // namespace amd diff --git a/src/hipamd/src/cl_gl_amd.hpp b/src/hipamd/src/cl_gl_amd.hpp deleted file mode 100644 index 36831fa747..0000000000 --- a/src/hipamd/src/cl_gl_amd.hpp +++ /dev/null @@ -1,379 +0,0 @@ -/* Copyright (c) 2010-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef CL_GL_AMD_HPP_ -#define CL_GL_AMD_HPP_ - -#ifdef _WIN32 -#include -#else //!_WIN32 -#include -#endif //!_WIN32 - -#include -#include -#include "CL/cl_gl.h" -#ifndef _WIN32 -#include -#endif //!_WIN32 - -#include -#include -#include - -#include "platform/context.hpp" -#include "platform/command.hpp" - -namespace amd -{ - -//! Class GLObject keeps all the info about the GL object -//! from which the CL object is created -class GLObject : public InteropObject -{ -protected: - cl_gl_object_type clGLType_; //!< CL GL object type - GLenum glTarget_; - GLuint gluiName_; - GLint gliMipLevel_; - GLenum glInternalFormat_; - GLint gliWidth_; - GLint gliHeight_; - GLint gliDepth_; - GLenum glCubemapFace_; - GLsizei glNumSamples_; - -public: -//! GLObject constructor initializes member variables - GLObject( - GLenum glTarget, - GLuint gluiName, - GLint gliMipLevel, - GLenum glInternalFormat, - GLint gliWidth, - GLint gliHeight, - GLint gliDepth, - cl_gl_object_type clGLType, - GLenum glCubemapFace, - GLsizei glNumSamples - ): // Initialization of member variables - clGLType_(clGLType), - glTarget_(glTarget), - gluiName_(gluiName), - gliMipLevel_(gliMipLevel), - glInternalFormat_(glInternalFormat), - gliWidth_(gliWidth), - gliHeight_(gliHeight), - gliDepth_(gliDepth), - glCubemapFace_(glCubemapFace), - glNumSamples_(glNumSamples) - { - } - - virtual ~GLObject() {} - virtual GLObject* asGLObject() {return this;} - -//! GLObject query functions to get GL info from member variables - GLenum getGLTarget() const {return glTarget_;} - GLuint getGLName() const {return gluiName_;} - GLint getGLMipLevel() const {return gliMipLevel_;} - GLenum getGLInternalFormat() const {return glInternalFormat_;} - GLint getGLSize() const {return gliWidth_;} - GLint getGLWidth() const {return gliWidth_;} - GLint getGLHeight() const {return gliHeight_;} - GLint getGLDepth() const {return gliDepth_;} - cl_gl_object_type getCLGLObjectType() const { return clGLType_; } - GLenum getCubemapFace() const {return glCubemapFace_;} - GLsizei getNumSamples() const { return glNumSamples_;} -}; - - -//! Class BufferGL is drived from classes Buffer and GLObject -//! where the former keeps all data for CL object and -//! the latter keeps all data for GL object -class BufferGL : public Buffer, public GLObject -{ -protected: - //! Initializes the device memory array which is nested - // after'BufferGL' object in memory layout. - virtual void initDeviceMemory(); -public: -//! BufferGL constructor just calls constructors of base classes -//! to pass down the parameters - BufferGL( - Context& amdContext, - cl_mem_flags clFlags, - size_t uiSizeInBytes, - GLenum glTarget, - GLuint gluiName) - : // Call base classes constructors - Buffer( - amdContext, - clFlags, - uiSizeInBytes - ), - GLObject( - glTarget, - gluiName, - 0, // Mipmap level default - GL_ARRAY_BUFFER, // Just init to some value - (GLint) uiSizeInBytes, - 1, - 1, - CL_GL_OBJECT_BUFFER, - 0, - 0 - ) - { - setInteropObj(this); - } - virtual ~BufferGL() {} - - virtual BufferGL* asBufferGL() { return this; } -}; - - -//! Class ImageGL is derived from classes Image and GLObject -//! where the former keeps all data for CL object and -//! the latter keeps all data for GL object -class ImageGL : public Image, public GLObject -{ -public: - //! ImageGL constructor just calls constructors of base classes - //! to pass down the parameters - ImageGL( - Context& amdContext, - cl_mem_object_type clType, - cl_mem_flags clFlags, - const Format& format, - size_t width, - size_t height, - size_t depth, - GLenum glTarget, - GLuint gluiName, - GLint gliMipLevel, - GLenum glInternalFormat, - cl_gl_object_type clGLType, - GLsizei numSamples, - GLenum glCubemapFace = 0) - : Image(amdContext, clType, clFlags, format, width, height, depth, - Format(format).getElementSize() * width, - Format(format).getElementSize() * width * depth) - , GLObject(glTarget, gluiName, gliMipLevel, glInternalFormat, - static_cast(width), static_cast(height), - static_cast(depth), clGLType, glCubemapFace,numSamples) - { - setInteropObj(this); - } - - virtual ~ImageGL() {} - -protected: - //! Initializes the device memory array which is nested - // after'BufferGL' object in memory layout. - virtual void initDeviceMemory(); -}; - -#ifdef _WIN32 -#define APICALL WINAPI -#define GETPROCADDRESS GetProcAddress -#define API_GETPROCADDR "wglGetProcAddress" -#define FCN_STR_TYPE LPCSTR - typedef PROC (WINAPI* PFN_xxxGetProcAddress) (LPCSTR fcnName); - typedef HGLRC (APICALL* PFN_wglCreateContext) (HDC hdc); - typedef HGLRC (APICALL* PFN_wglGetCurrentContext) (void); - typedef HDC (APICALL* PFN_wglGetCurrentDC) (void); - typedef BOOL (APICALL* PFN_wglDeleteContext) (HGLRC hglrc); - typedef BOOL (APICALL* PFN_wglMakeCurrent) (HDC hdc, HGLRC hglrc); - typedef BOOL (APICALL* PFN_wglShareLists) (HGLRC hglrc1, HGLRC hglrc2); -#else //!_WIN32 -#define APICALL // __stdcall //??? todo odintsov -#define API_GETPROCADDR "glXGetProcAddress" -#define GETPROCADDRESS dlsym -#define FCN_STR_TYPE const GLubyte* -#define WINAPI -#define PROC void* - typedef void* (*PFN_xxxGetProcAddress) (const GLubyte* procName); - // X11 typedef - typedef Display* (*PFNXOpenDisplay)(_Xconst char* display_name ); - typedef int (*PFNXCloseDisplay)(Display* display ); - - //glx typedefs - typedef GLXDrawable (*PFNglXGetCurrentDrawable)(); - typedef Display* (*PFNglXGetCurrentDisplay)(); - typedef GLXContext (*PFNglXGetCurrentContext)( void ); - typedef XVisualInfo* (*PFNglXChooseVisual)(Display *dpy, int screen, int *attribList); - typedef GLXContext(*PFNglXCreateContext)(Display* dpy,XVisualInfo* vis,GLXContext shareList,Bool direct); - typedef void(*PFNglXDestroyContext)(Display* dpy, GLXContext ctx); - typedef Bool(*PFNglXMakeCurrent)( Display* dpy, GLXDrawable drawable, GLXContext ctx); - typedef void* HMODULE; -#endif //!_WIN32 - -#define GLPREFIX(rtype, fcn, dclargs) \ - typedef rtype (APICALL* PFN_##fcn) dclargs; - -// Declare prototypes for GL functions -#include "gl_functions.hpp" - -class GLFunctions -{ -public: - //! Locks any access to the virtual GPUs - class SetIntEnv : public amd::StackObject { - public: - //! Default constructor - SetIntEnv(GLFunctions* env); - - //! Destructor - ~SetIntEnv(); - - //! Checks if the environment setup was successful - bool isValid() const { return isValid_; } - - private: - GLFunctions* env_; //!< GL environment - bool isValid_; //!< If TRUE, then it's a valid setup - }; - -private: - HMODULE libHandle_; - int missed_; // Indicates how many GL functions not init'ed, if any - - amd::Monitor lock_; - - EGLDisplay eglDisplay_; - EGLContext eglOriginalContext_; - EGLContext eglInternalContext_; - EGLContext eglTempContext_; - bool isEGL_; - -#ifdef _WIN32 - HGLRC hOrigGLRC_; - HDC hDC_; - HGLRC hIntGLRC_; // handle for internal GLRC to access shared context - HDC tempDC_; - HGLRC tempGLRC_; - - PFN_wglCreateContext wglCreateContext_; - PFN_wglGetCurrentContext wglGetCurrentContext_; - PFN_wglGetCurrentDC wglGetCurrentDC_; - PFN_wglDeleteContext wglDeleteContext_; - PFN_wglMakeCurrent wglMakeCurrent_; - PFN_wglShareLists wglShareLists_; -#else -public: - Display* Dpy_; - GLXDrawable Drawable_; - GLXContext origCtx_; - Display* intDpy_; - Window intDrawable_; - GLXContext intCtx_; - Display* tempDpy_; - GLXDrawable tempDrawable_; - GLXContext tempCtx_; - - //pointers to X11 functions - PFNXOpenDisplay XOpenDisplay_; - PFNXCloseDisplay XCloseDisplay_; - - //pointers to GLX functions - PFNglXGetCurrentDrawable glXGetCurrentDrawable_; - PFNglXGetCurrentDisplay glXGetCurrentDisplay_; - PFNglXGetCurrentContext glXGetCurrentContext_; - PFNglXChooseVisual glXChooseVisual_; - PFNglXCreateContext glXCreateContext_; - PFNglXDestroyContext glXDestroyContext_; - PFNglXMakeCurrent glXMakeCurrent_; -#endif -public: - - GLFunctions(HMODULE h, bool isEGL); - ~GLFunctions(); - - // Query CL-GL context association - bool isAssociated() const - { - if (isEGL_ && eglDisplay_ && eglOriginalContext_) return true; -#ifdef _WIN32 - if(hDC_ && hOrigGLRC_) return true; -#else //!_WIN32 - if(Dpy_ && origCtx_) return true; -#endif //!_WIN32 - return false; - } - bool isEGL() const - { - return isEGL_; - } - // Accessor methods -#ifdef _WIN32 - HGLRC getOrigGLRC() const {return hOrigGLRC_;} - HDC getDC() const {return hDC_;} - HGLRC getIntGLRC() const {return hIntGLRC_;} -#else //!_WIN32 - Display* getDpy() const {return Dpy_;} - GLXDrawable getDrawable() const {return Drawable_;} - GLXContext getOrigCtx() const {return origCtx_;} - - Display* getIntDpy() const {return intDpy_;} - GLXDrawable getIntDrawable() const {return intDrawable_;} - GLXContext getIntCtx() const {return intCtx_;} - - EGLDisplay getEglDpy() const { return eglDisplay_; } - EGLContext getEglOrigCtx() const { return eglOriginalContext_; } -#endif //!_WIN32 - - // Initialize GL dynamic library and function pointers - bool init(intptr_t hdc, intptr_t hglrc); - - // Return true if successful, false - if error occurred - bool setIntEnv(); - bool restoreEnv(); - - amd::Monitor& getLock() { return lock_; } - - PFN_xxxGetProcAddress GetProcAddress_; - -#define GLPREFIX(rtype, fcn, dclargs) \ - PFN_##fcn fcn##_; -// Declare pointers to GL functions -#include "gl_functions.hpp" -}; - -//! Functions for executing the GL related stuff -cl_mem clCreateFromGLBufferAMD(Context& amdContext, cl_mem_flags flags, - GLuint bufobj, cl_int* errcode_ret); -cl_mem clCreateFromGLTextureAMD(Context& amdContext, cl_mem_flags flags, - GLenum target, GLint miplevel, GLuint texture, int* errcode_ret); -cl_mem clCreateFromGLRenderbufferAMD(Context& amdContext, cl_mem_flags flags, - GLuint renderbuffer, int* errcode_ret); - -bool -getCLFormatFromGL( - const Context& amdContext, - GLint gliInternalFormat, - cl_image_format* pclImageFormat, - int* piBytesPerPixel, - cl_mem_flags flags -); - -} //namespace amd - -#endif //CL_GL_AMD_HPP_ diff --git a/src/hipamd/src/cl_lqdflash_amd.cpp b/src/hipamd/src/cl_lqdflash_amd.cpp deleted file mode 100644 index 470e4bc5e9..0000000000 --- a/src/hipamd/src/cl_lqdflash_amd.cpp +++ /dev/null @@ -1,312 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "cl_common.hpp" -#include - -#include "platform/object.hpp" - -#include "cl_lqdflash_amd.h" - -#ifndef WITH_LIQUID_FLASH -#if (!defined(BUILD_HSA_TARGET) && defined(WITH_HSA_DEVICE) && \ - defined(WITH_AMDGPU_PRO)) || defined(_WIN32) -#define WITH_LIQUID_FLASH 1 -#endif // _WIN32 -#endif - -#if WITH_LIQUID_FLASH -#include "lf.h" -#include -#include -#endif // WITH_LIQUID_FLASH - -namespace amd { - -LiquidFlashFile::~LiquidFlashFile() { close(); } - -bool LiquidFlashFile::open() { -#if WITH_LIQUID_FLASH - lf_status err; - lf_file_flags flags = 0; - - switch (flags_) { - case CL_FILE_READ_ONLY_AMD: - flags = LF_READ; - break; - case CL_FILE_WRITE_ONLY_AMD: - flags = LF_WRITE; - break; - case CL_FILE_READ_WRITE_AMD: - flags = LF_READ | LF_WRITE; - break; - } -#ifdef ATI_OS_LINUX - assert(sizeof(wchar_t) != sizeof(lf_char)); - std::string name_char; - std::wstring_convert, wchar_t> cv; - name_char = cv.to_bytes(name_); - handle_ = lfOpenFile(name_char.c_str(), flags, &err); -#else - handle_ = lfOpenFile(name_.c_str(), flags, &err); -#endif - - if (err != lf_success) { - return false; - } - - if (lfGetFileBlockSize((lf_file)handle_, &blockSize_) != lf_success) { - return false; - } - - if (lfGetFileSize((lf_file)handle_, &fileSize_) != lf_success) { - return false; - } - return true; -#else - return false; -#endif // WITH_LIQUID_FLASH -} - -void LiquidFlashFile::close() { -#if WITH_LIQUID_FLASH - if (handle_ != NULL) { - lfReleaseFile((lf_file)handle_); - handle_ = NULL; - } -#endif // WITH_LIQUID_FLASH -} - -bool LiquidFlashFile::transferBlock(bool writeBuffer, void* srcDst, uint64_t bufferSize, - uint64_t fileOffset, uint64_t bufferOffset, - uint64_t size) const { -#if WITH_LIQUID_FLASH - lf_status status; - - lf_region_descriptor region = {fileOffset / blockSize(), bufferOffset / blockSize(), - size / blockSize()}; - if (writeBuffer) { - status = lfReadFile(srcDst, bufferSize, (lf_file)handle_, 1, ®ion, NULL); - } else { - status = lfWriteFile(srcDst, bufferSize, (lf_file)handle_, 1, ®ion, NULL); - } - if (lf_success == status) { - return true; - } else { - return false; - } -#else - return false; -#endif // WITH_LIQUID_FLASH -} - -} // namespace amd - -/*! \addtogroup API - * @{ - * - * \addtogroup AMD_Extensions - * @{ - * - */ - -RUNTIME_ENTRY_RET(cl_file_amd, clCreateSsgFileObjectAMD, - (cl_context context, cl_file_flags_amd flags, const wchar_t* file_name, - cl_int* errcode_ret)) { -#if WITH_LIQUID_FLASH && defined ATI_OS_LINUX - if (!is_valid(context)) { - *not_null(errcode_ret) = CL_INVALID_CONTEXT; - LogWarning("invalid parameter \"context\""); - return (cl_file_amd)0; - } - - const std::vector& devices = as_amd(context)->devices(); - bool supportPass = false; - for (auto& dev : devices) { - if (lf_success == lfCheckExtensionSupportForDevice(dev->info().pcieDeviceId_, - dev->info().pcieRevisionId_)) { - supportPass = true; - break; - } - } - if (!supportPass) { - *not_null(errcode_ret) = CL_INVALID_DEVICE; - LogWarning("SSG isn't supported"); - return (cl_file_amd)0; - } -#endif - amd::LiquidFlashFile* file = new amd::LiquidFlashFile(file_name, flags); - - if (file == NULL) { - *not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY; - return (cl_file_amd)0; - } - - if (!file->open()) { - *not_null(errcode_ret) = CL_INVALID_VALUE; - delete file; - return (cl_file_amd)0; - } - - *not_null(errcode_ret) = CL_SUCCESS; - return as_cl(file); -} -RUNTIME_EXIT - -RUNTIME_ENTRY(cl_int, clGetSsgFileObjectInfoAMD, - (cl_file_amd file, cl_file_info_amd param_name, size_t param_value_size, - void* param_value, size_t* param_value_size_ret)) { - if (!is_valid(file)) { - return CL_INVALID_FILE_OBJECT_AMD; - } - - switch (param_name) { - case CL_FILE_BLOCK_SIZE_AMD: { - cl_uint blockSize = as_amd(file)->blockSize(); - return amd::clGetInfo(blockSize, param_value_size, param_value, param_value_size_ret); - } - case CL_FILE_SIZE_AMD: { - cl_ulong fileSize = as_amd(file)->fileSize(); - return amd::clGetInfo(fileSize, param_value_size, param_value, param_value_size_ret); - } - default: - break; - } - - return CL_INVALID_VALUE; -} -RUNTIME_EXIT - -RUNTIME_ENTRY(cl_int, clRetainSsgFileObjectAMD, (cl_file_amd file)) { - if (!is_valid(file)) { - return CL_INVALID_FILE_OBJECT_AMD; - } - as_amd(file)->retain(); - return CL_SUCCESS; -} -RUNTIME_EXIT - -RUNTIME_ENTRY(cl_int, clReleaseSsgFileObjectAMD, (cl_file_amd file)) { - if (!is_valid(file)) { - return CL_INVALID_FILE_OBJECT_AMD; - } - as_amd(file)->release(); - return CL_SUCCESS; -} -RUNTIME_EXIT - -static cl_int EnqueueTransferBufferFromSsgFileAMD( - cl_bool isWrite, cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, - size_t buffer_offset, size_t cb, cl_file_amd file, size_t file_offset, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event) { - if (!is_valid(command_queue)) { - return CL_INVALID_COMMAND_QUEUE; - } - - if (!is_valid(buffer)) { - return CL_INVALID_MEM_OBJECT; - } - amd::Buffer* pBuffer = as_amd(buffer)->asBuffer(); - if (pBuffer == NULL) { - return CL_INVALID_MEM_OBJECT; - } - - if (pBuffer->getMemFlags() & (CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS)) { - return CL_INVALID_OPERATION; - } - - amd::HostQueue* queue = as_amd(command_queue)->asHostQueue(); - if (NULL == queue) { - return CL_INVALID_COMMAND_QUEUE; - } - amd::HostQueue& hostQueue = *queue; - - if (hostQueue.context() != pBuffer->getContext()) { - return CL_INVALID_CONTEXT; - } - - if (!is_valid(file)) { - return CL_INVALID_FILE_OBJECT_AMD; - } - - amd::LiquidFlashFile* amdFile = as_amd(file); - amd::Coord3D bufferOffset(buffer_offset, 0, 0); - amd::Coord3D bufferSize(cb, 1, 1); - - if ((!pBuffer->validateRegion(bufferOffset, bufferSize)) || - // LF library supports aligned sizes only - ((buffer_offset % amdFile->blockSize()) != 0) || ((cb % amdFile->blockSize()) != 0) || - ((file_offset % amdFile->blockSize()) != 0)) { - return CL_INVALID_VALUE; - } - - amd::Command::EventWaitList eventWaitList; - cl_int err = amd::clSetEventWaitList(eventWaitList, hostQueue, num_events_in_wait_list, - event_wait_list); - if (err != CL_SUCCESS) { - return err; - } - - amd::TransferBufferFileCommand* command; - command = new amd::TransferBufferFileCommand( - isWrite ? CL_COMMAND_READ_SSG_FILE_AMD : CL_COMMAND_WRITE_SSG_FILE_AMD, hostQueue, - eventWaitList, *pBuffer, bufferOffset, bufferSize, amdFile, file_offset); - - if (command == NULL) { - return CL_OUT_OF_HOST_MEMORY; - } - - // Make sure we have memory for the command execution - if (!command->validateMemory()) { - delete command; - return CL_MEM_OBJECT_ALLOCATION_FAILURE; - } - - command->enqueue(); - if (blocking_write) { - command->awaitCompletion(); - } - - *not_null(event) = as_cl(&command->event()); - if (event == NULL) { - command->release(); - } - return CL_SUCCESS; -} - -RUNTIME_ENTRY(cl_int, clEnqueueReadSsgFileAMD, - (cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, - size_t buffer_offset, size_t cb, cl_file_amd file, size_t file_offset, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return EnqueueTransferBufferFromSsgFileAMD(CL_TRUE, command_queue, buffer, blocking_write, - buffer_offset, cb, file, file_offset, - num_events_in_wait_list, event_wait_list, event); -} -RUNTIME_EXIT - -RUNTIME_ENTRY(cl_int, clEnqueueWriteSsgFileAMD, - (cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, - size_t buffer_offset, size_t cb, cl_file_amd file, size_t file_offset, - cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) { - return EnqueueTransferBufferFromSsgFileAMD(CL_FALSE, command_queue, buffer, blocking_write, - buffer_offset, cb, file, file_offset, - num_events_in_wait_list, event_wait_list, event); -} -RUNTIME_EXIT diff --git a/src/hipamd/src/cl_lqdflash_amd.h b/src/hipamd/src/cl_lqdflash_amd.h deleted file mode 100644 index 5a3e725b4c..0000000000 --- a/src/hipamd/src/cl_lqdflash_amd.h +++ /dev/null @@ -1,58 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef __CL_LQDFLASH_AMD_H -#define __CL_LQDFLASH_AMD_H - -#include "CL/cl_ext.h" - -#ifdef __cplusplus -extern "C" { -#endif /*__cplusplus*/ - -extern CL_API_ENTRY cl_file_amd CL_API_CALL -clCreateSsgFileObjectAMD(cl_context context, cl_file_flags_amd flags, const wchar_t* file_name, - cl_int* errcode_ret) CL_EXT_SUFFIX__VERSION_1_2; - -extern CL_API_ENTRY cl_int CL_API_CALL clGetSsgFileObjectInfoAMD( - cl_file_amd file, cl_file_info_amd param_name, size_t param_value_size, void* param_value, - size_t* param_value_size_ret) CL_EXT_SUFFIX__VERSION_1_2; - -extern CL_API_ENTRY cl_int CL_API_CALL clRetainSsgFileObjectAMD(cl_file_amd file) - CL_EXT_SUFFIX__VERSION_1_2; - -extern CL_API_ENTRY cl_int CL_API_CALL clReleaseSsgFileObjectAMD(cl_file_amd file) - CL_EXT_SUFFIX__VERSION_1_2; - -extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueReadSsgFileAMD( - cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, size_t buffer_offset, - size_t cb, cl_file_amd file, size_t file_offset, cl_uint num_events_in_wait_list, - const cl_event* event_wait_list, cl_event* event) CL_EXT_SUFFIX__VERSION_1_2; - -extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueWriteSsgFileAMD( - cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, size_t buffer_offset, - size_t cb, cl_file_amd file, size_t file_offset, cl_uint num_events_in_wait_list, - const cl_event* event_wait_list, cl_event* event) CL_EXT_SUFFIX__VERSION_1_2; - -#ifdef __cplusplus -} /*extern "C"*/ -#endif /*__cplusplus*/ - -#endif diff --git a/src/hipamd/src/fixme.cpp b/src/hipamd/src/fixme.cpp deleted file mode 100644 index 90f034f63e..0000000000 --- a/src/hipamd/src/fixme.cpp +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "vdi_common.hpp" -#include - -cl_icd_dispatch amd::ICDDispatchedObject::icdVendorDispatch_[] = {0}; -amd::PlatformIDS amd::PlatformID::Platform = {amd::ICDDispatchedObject::icdVendorDispatch_}; - -RUNTIME_ENTRY(cl_int, clGetDeviceIDs, - (cl_platform_id platform, cl_device_type device_type, cl_uint num_entries, - cl_device_id* devices, cl_uint* num_devices)) { - return CL_SUCCESS; -} -RUNTIME_EXIT diff --git a/src/hipamd/src/hip_activity.cpp b/src/hipamd/src/hip_activity.cpp deleted file mode 100644 index d3ce84bfd9..0000000000 --- a/src/hipamd/src/hip_activity.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "platform/activity.hpp" - -extern "C" void hipInitActivityCallback(void* id_callback, void* op_callback, void* arg) { - activity_prof::CallbacksTable::init(reinterpret_cast(id_callback), - reinterpret_cast(op_callback), - arg); -} - -extern "C" bool hipEnableActivityCallback(unsigned op, bool enable) { - return activity_prof::CallbacksTable::SetEnabled(op, enable); -} - -extern "C" const char* hipGetCmdName(unsigned op) { - return getOclCommandKindString(static_cast(op)); -} diff --git a/src/hipamd/src/hip_code_object.cpp b/src/hipamd/src/hip_code_object.cpp deleted file mode 100755 index 8bd790f1f3..0000000000 --- a/src/hipamd/src/hip_code_object.cpp +++ /dev/null @@ -1,850 +0,0 @@ -/* -Copyright (c) 2015-2020 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ -#include "hip_code_object.hpp" -#include "amd_hsa_elf.hpp" - -#include - -#include -#include "hip/hip_runtime_api.h" -#include "hip/hip_runtime.h" -#include "hip_internal.hpp" -#include "platform/program.hpp" -#include - -hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind, - amd::HostQueue& queue, bool isAsync = false); -hipError_t ihipFree(void* ptr); -//forward declaration of methods required for managed variables -hipError_t ihipMallocManaged(void** ptr, size_t size, unsigned int align = 0); -namespace { -size_t constexpr strLiteralLength(char const* str) { - return *str ? 1 + strLiteralLength(str + 1) : 0; -} -constexpr char const* CLANG_OFFLOAD_BUNDLER_MAGIC_STR = "__CLANG_OFFLOAD_BUNDLE__"; -constexpr char const* OFFLOAD_KIND_HIP = "hip"; -constexpr char const* OFFLOAD_KIND_HIPV4 = "hipv4"; -constexpr char const* OFFLOAD_KIND_HCC = "hcc"; -constexpr char const* AMDGCN_TARGET_TRIPLE = "amdgcn-amd-amdhsa-"; - -// ClangOFFLOADBundle info. -static constexpr size_t bundle_magic_string_size = - strLiteralLength(CLANG_OFFLOAD_BUNDLER_MAGIC_STR); - -// Clang Offload bundler description & Header. -struct __ClangOffloadBundleInfo { - uint64_t offset; - uint64_t size; - uint64_t bundleEntryIdSize; - const char bundleEntryId[1]; -}; - -struct __ClangOffloadBundleHeader { - const char magic[bundle_magic_string_size - 1]; - uint64_t numOfCodeObjects; - __ClangOffloadBundleInfo desc[1]; -}; -} // namespace - -namespace hip { - -uint64_t CodeObject::ElfSize(const void *emi) { - return amd::Elf::getElfSize(emi); -} - -static bool getProcName(uint32_t EFlags, std::string& proc_name, bool& xnackSupported, - bool& sramEccSupported) { - switch (EFlags & EF_AMDGPU_MACH) { - case EF_AMDGPU_MACH_AMDGCN_GFX700: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx700"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX701: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx701"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX702: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx702"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX703: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx703"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX704: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx704"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX705: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx705"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX801: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx801"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX802: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx802"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX803: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx803"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX805: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx805"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX810: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx810"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX900: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx900"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX902: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx902"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX904: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx904"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX906: - xnackSupported = true; - sramEccSupported = true; - proc_name = "gfx906"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX908: - xnackSupported = true; - sramEccSupported = true; - proc_name = "gfx908"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX909: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx909"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX90A: - xnackSupported = true; - sramEccSupported = true; - proc_name = "gfx90a"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX90C: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx90c"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX1010: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx1010"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX1011: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx1011"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX1012: - xnackSupported = true; - sramEccSupported = false; - proc_name = "gfx1012"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX1030: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx1030"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX1031: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx1031"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX1032: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx1032"; - break; - case EF_AMDGPU_MACH_AMDGCN_GFX1033: - xnackSupported = false; - sramEccSupported = false; - proc_name = "gfx1033"; - break; - default: - return false; - } - return true; -} - -static bool getTripleTargetIDFromCodeObject(const void* code_object, std::string& target_id, - unsigned& co_version) { - if (!code_object) return false; - const Elf64_Ehdr* ehdr = reinterpret_cast(code_object); - if (ehdr->e_machine != EM_AMDGPU) return false; - if (ehdr->e_ident[EI_OSABI] != ELFOSABI_AMDGPU_HSA) return false; - - bool isXnackSupported{false}, isSramEccSupported{false}; - - std::string proc_name; - if (!getProcName(ehdr->e_flags, proc_name, isXnackSupported, isSramEccSupported)) return false; - target_id = std::string(AMDGCN_TARGET_TRIPLE) + '-' + proc_name; - - switch (ehdr->e_ident[EI_ABIVERSION]) { - case ELFABIVERSION_AMDGPU_HSA_V2: { - co_version = 2; - return false; - } - - case ELFABIVERSION_AMDGPU_HSA_V3: { - co_version = 3; - if (isSramEccSupported) { - if (ehdr->e_flags & EF_AMDGPU_FEATURE_SRAMECC_V3) - target_id += ":sramecc+"; - else - target_id += ":sramecc-"; - } - if (isXnackSupported) { - if (ehdr->e_flags & EF_AMDGPU_FEATURE_XNACK_V3) - target_id += ":xnack+"; - else - target_id += ":xnack-"; - } - break; - } - - case ELFABIVERSION_AMDGPU_HSA_V4: { - co_version = 4; - unsigned co_sram_value = (ehdr->e_flags) & EF_AMDGPU_FEATURE_SRAMECC_V4; - if (co_sram_value == EF_AMDGPU_FEATURE_SRAMECC_OFF_V4) - target_id += ":sramecc-"; - else if (co_sram_value == EF_AMDGPU_FEATURE_SRAMECC_ON_V4) - target_id += ":sramecc+"; - - unsigned co_xnack_value = (ehdr->e_flags) & EF_AMDGPU_FEATURE_XNACK_V4; - if (co_xnack_value == EF_AMDGPU_FEATURE_XNACK_OFF_V4) - target_id += ":xnack-"; - else if (co_xnack_value == EF_AMDGPU_FEATURE_XNACK_ON_V4) - target_id += ":xnack+"; - break; - } - - default: { - return false; - } - } - return true; -} - -// Consumes the string 'consume_' from the starting of the given input -// eg: input = amdgcn-amd-amdhsa--gfx908 and consume_ is amdgcn-amd-amdhsa-- -// input will become gfx908. -static bool consume(std::string& input, std::string consume_) { - if (input.substr(0, consume_.size()) != consume_) { - return false; - } - input = input.substr(consume_.size()); - return true; -} - -// Trim String till character, will be used to get gpuname -// example: input is gfx908:sram-ecc+ and trim char is : -// input will become sram-ecc+. -static std::string trimName(std::string& input, char trim) { - auto pos_ = input.find(trim); - auto res = input; - if (pos_ == std::string::npos) { - input = ""; - } else { - res = input.substr(0, pos_); - input = input.substr(pos_); - } - return res; -} - -static char getFeatureValue(std::string& input, std::string feature) { - char res = ' '; - if (consume(input, std::move(feature))) { - res = input[0]; - input = input.substr(1); - } - return res; -} - -static bool getTargetIDValue(std::string& input, std::string& processor, char& sramecc_value, - char& xnack_value) { - processor = trimName(input, ':'); - sramecc_value = getFeatureValue(input, std::string(":sramecc")); - if (sramecc_value != ' ' && sramecc_value != '+' && sramecc_value != '-') return false; - xnack_value = getFeatureValue(input, std::string(":xnack")); - if (xnack_value != ' ' && xnack_value != '+' && xnack_value != '-') return false; - return true; -} - -static bool getTripleTargetID(std::string bundled_co_entry_id, const void* code_object, - std::string& co_triple_target_id, unsigned& co_version) { - std::string offload_kind = trimName(bundled_co_entry_id, '-'); - if (offload_kind != OFFLOAD_KIND_HIPV4 && offload_kind != OFFLOAD_KIND_HIP && - offload_kind != OFFLOAD_KIND_HCC) - return false; - - if (offload_kind != OFFLOAD_KIND_HIPV4) - return getTripleTargetIDFromCodeObject(code_object, co_triple_target_id, co_version); - - // For code object V4 onwards the bundled code object entry ID correctly - // specifies the target tripple. - co_version = 4; - co_triple_target_id = bundled_co_entry_id.substr(1); - return true; -} - -static bool isCodeObjectCompatibleWithDevice(std::string co_triple_target_id, - std::string agent_triple_target_id) { - // Primitive Check - if (co_triple_target_id == agent_triple_target_id) return true; - - // Parse code object triple target id - if (!consume(co_triple_target_id, std::string(AMDGCN_TARGET_TRIPLE) + '-')) { - return false; - } - - std::string co_processor; - char co_sram_ecc, co_xnack; - if (!getTargetIDValue(co_triple_target_id, co_processor, co_sram_ecc, co_xnack)) { - return false; - } - - if (!co_triple_target_id.empty()) return false; - - // Parse agent isa triple target id - if (!consume(agent_triple_target_id, std::string(AMDGCN_TARGET_TRIPLE) + '-')) { - return false; - } - - std::string agent_isa_processor; - char isa_sram_ecc, isa_xnack; - if (!getTargetIDValue(agent_triple_target_id, agent_isa_processor, isa_sram_ecc, isa_xnack)) { - return false; - } - - if (!agent_triple_target_id.empty()) return false; - - // Check for compatibility - if (agent_isa_processor != co_processor) return false; - if (co_sram_ecc != ' ') { - if (co_sram_ecc != isa_sram_ecc) return false; - } - if (co_xnack != ' ') { - if (co_xnack != isa_xnack) return false; - } - - return true; -} - -// This will be moved to COMGR eventually -hipError_t CodeObject::ExtractCodeObjectFromFile(amd::Os::FileDesc fdesc, size_t fsize, - const void ** image, const std::vector& device_names, - std::vector>& code_objs) { - - hipError_t hip_error = hipSuccess; - - if (fdesc < 0) { - return hipErrorFileNotFound; - } - - // Map the file to memory, with offset 0. - //file will be unmapped in ModuleUnload - //const void* image = nullptr; - if (!amd::Os::MemoryMapFileDesc(fdesc, fsize, 0, image)) { - return hipErrorInvalidValue; - } - - // retrieve code_objs{binary_image, binary_size} for devices - hip_error = extractCodeObjectFromFatBinary(*image, device_names, code_objs); - - return hip_error; -} - -// This will be moved to COMGR eventually -hipError_t CodeObject::ExtractCodeObjectFromMemory(const void* data, - const std::vector& device_names, - std::vector>& code_objs, - std::string& uri) { - - // Get the URI from memory - if (!amd::Os::GetURIFromMemory(data, 0, uri)) { - return hipErrorInvalidValue; - } - - return extractCodeObjectFromFatBinary(data, device_names, code_objs); -} - -// This will be moved to COMGR eventually -hipError_t CodeObject::extractCodeObjectFromFatBinary(const void* data, - const std::vector& agent_triple_target_ids, - std::vector>& code_objs) { - std::string magic((const char*)data, bundle_magic_string_size); - if (magic.compare(CLANG_OFFLOAD_BUNDLER_MAGIC_STR)) { - return hipErrorInvalidKernelFile; - } - - // Initialize Code objects - code_objs.reserve(agent_triple_target_ids.size()); - for (size_t i = 0; i < agent_triple_target_ids.size(); i++) { - code_objs.push_back(std::make_pair(nullptr, 0)); - } - - const auto obheader = reinterpret_cast(data); - const auto* desc = &obheader->desc[0]; - size_t num_code_objs = code_objs.size(); - for (uint64_t i = 0; i < obheader->numOfCodeObjects; ++i, - desc = reinterpret_cast( - reinterpret_cast(&desc->bundleEntryId[0]) + - desc->bundleEntryIdSize)) { - const void* image = - reinterpret_cast(reinterpret_cast(obheader) + desc->offset); - const size_t image_size = desc->size; - - if (num_code_objs == 0) break; - std::string bundleEntryId{desc->bundleEntryId, desc->bundleEntryIdSize}; - - unsigned co_version = 0; - std::string co_triple_target_id; - if (!getTripleTargetID(bundleEntryId, image, co_triple_target_id, co_version)) continue; - - for (size_t dev = 0; dev < agent_triple_target_ids.size(); ++dev) { - if (code_objs[dev].first) continue; - if (isCodeObjectCompatibleWithDevice(co_triple_target_id, agent_triple_target_ids[dev])) { - code_objs[dev] = std::make_pair(image, image_size); - --num_code_objs; - } - } - } - if (num_code_objs == 0) { - return hipSuccess; - } else { - LogPrintfError("%s", - "hipErrorNoBinaryForGpu: Unable to find code object for all current devices!"); - LogPrintfError("%s", " Devices:"); - for (size_t i = 0; i < agent_triple_target_ids.size(); i++) { - LogPrintfError(" %s - [%s]", agent_triple_target_ids[i].c_str(), - ((code_objs[i].first) ? "Found" : "Not Found")); - } - const auto obheader = reinterpret_cast(data); - const auto* desc = &obheader->desc[0]; - LogPrintfError("%s", " Bundled Code Objects:"); - for (uint64_t i = 0; i < obheader->numOfCodeObjects; ++i, - desc = reinterpret_cast( - reinterpret_cast(&desc->bundleEntryId[0]) + - desc->bundleEntryIdSize)) { - std::string bundleEntryId{desc->bundleEntryId, desc->bundleEntryIdSize}; - const void* image = - reinterpret_cast(reinterpret_cast(obheader) + desc->offset); - - unsigned co_version = 0; - std::string co_triple_target_id; - bool valid_co = getTripleTargetID(bundleEntryId, image, co_triple_target_id, co_version); - - if (valid_co) { - LogPrintfError(" %s - [code object v%u is %s]", bundleEntryId.c_str(), co_version, - co_triple_target_id.c_str()); - } else { - LogPrintfError(" %s - [Unsupported]", bundleEntryId.c_str()); - } - } - - guarantee(false, "hipErrorNoBinaryForGpu: Unable to find code object for all current devices!"); - return hipErrorNoBinaryForGpu; - } -} - -hipError_t DynCO::loadCodeObject(const char* fname, const void* image) { - - amd::ScopedLock lock(dclock_); - - // Number of devices = 1 in dynamic code object - fb_info_ = new FatBinaryInfo(fname, image); - std::vector devices = { g_devices[ihipGetDevice()] }; - IHIP_RETURN_ONFAIL(fb_info_->ExtractFatBinary(devices)); - - // No Lazy loading for DynCO - IHIP_RETURN_ONFAIL(fb_info_->BuildProgram(ihipGetDevice())); - - // Define Global variables - IHIP_RETURN_ONFAIL(populateDynGlobalVars()); - - // Define Global functions - IHIP_RETURN_ONFAIL(populateDynGlobalFuncs()); - - return hipSuccess; -} - -//Dynamic Code Object -DynCO::~DynCO() { - amd::ScopedLock lock(dclock_); - - for (auto& elem : vars_) { - if(elem.second->getVarKind() == Var::DVK_Managed) { - ihipFree(elem.second->getManagedVarPtr()); - } - delete elem.second; - } - vars_.clear(); - - for (auto& elem : functions_) { - delete elem.second; - } - functions_.clear(); - - delete fb_info_; -} - -hipError_t DynCO::getDeviceVar(DeviceVar** dvar, std::string var_name) { - amd::ScopedLock lock(dclock_); - - CheckDeviceIdMatch(); - - auto it = vars_.find(var_name); - if (it == vars_.end()) { - LogPrintfError("Cannot find the Var: %s ", var_name.c_str()); - return hipErrorNotFound; - } - - it->second->getDeviceVar(dvar, device_id_, module()); - return hipSuccess; -} - -hipError_t DynCO::getDynFunc(hipFunction_t* hfunc, std::string func_name) { - amd::ScopedLock lock(dclock_); - - CheckDeviceIdMatch(); - - if(hfunc == nullptr) { - return hipErrorInvalidValue; - } - - auto it = functions_.find(func_name); - if (it == functions_.end()) { - LogPrintfError("Cannot find the function: %s ", func_name.c_str()); - return hipErrorNotFound; - } - - /* See if this could be solved */ - return it->second->getDynFunc(hfunc, module()); -} - -hipError_t DynCO::initDynManagedVars(const std::string& managedVar) { - amd::ScopedLock lock(dclock_); - DeviceVar* dvar; - void* pointer = nullptr; - hipError_t status = hipSuccess; - // To get size of the managed variable - status = getDeviceVar(&dvar, managedVar + ".managed"); - if (status != hipSuccess) { - ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to get .managed device variable:%s", - status, managedVar.c_str()); - return status; - } - // Allocate managed memory for these symbols - status = ihipMallocManaged(&pointer, dvar->size()); - if (status != hipSuccess) { - ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to allocate managed memory", status); - guarantee(false, "Error during allocation of managed memory!"); - } - // update as manager variable and set managed memory pointer and size - auto it = vars_.find(managedVar); - it->second->setManagedVarInfo(pointer, dvar->size()); - - // copy initial value to the managed variable to the managed memory allocated - amd::HostQueue* queue = hip::getNullStream(); - if (queue != nullptr) { - status = ihipMemcpy(pointer, reinterpret_cast
(dvar->device_ptr()), dvar->size(), - hipMemcpyDeviceToDevice, *queue); - if (status != hipSuccess) { - ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to copy device ptr:%s", status, - managedVar.c_str()); - return status; - } - } else { - ClPrint(amd::LOG_ERROR, amd::LOG_API, "Host Queue is NULL"); - return hipErrorInvalidResourceHandle; - } - - // Get deivce ptr to initialize with managed memory pointer - status = getDeviceVar(&dvar, managedVar); - if (status != hipSuccess) { - ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to get managed device variable:%s", - status, managedVar.c_str()); - return status; - } - // copy managed memory pointer to the managed device variable - status = ihipMemcpy(reinterpret_cast
(dvar->device_ptr()), &pointer, dvar->size(), - hipMemcpyHostToDevice, *queue); - if (status != hipSuccess) { - ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to copy device ptr:%s", status, - managedVar.c_str()); - return status; - } - return status; -} - -hipError_t DynCO::populateDynGlobalVars() { - amd::ScopedLock lock(dclock_); - std::vector var_names; - std::vector undef_var_names; - std::string managedVarExt = ".managed"; - // For Dynamic Modules there is only one hipFatBinaryDevInfo_ - device::Program* dev_program = fb_info_->GetProgram(ihipGetDevice()) - ->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]); - - if (!dev_program->getGlobalVarFromCodeObj(&var_names)) { - LogPrintfError("Could not get Global vars from Code Obj for Module: 0x%x \n", module()); - return hipErrorSharedObjectSymbolNotFound; - } - - for (auto& elem : var_names) { - vars_.insert( - std::make_pair(elem, new Var(elem, Var::DeviceVarKind::DVK_Variable, 0, 0, 0, nullptr))); - } - - for (auto& elem : var_names) { - if (elem.find(managedVarExt) != std::string::npos) { - std::string managedVar = elem; - managedVar.erase(managedVar.length() - managedVarExt.length(), managedVarExt.length()); - initDynManagedVars(managedVar); - } - } - return hipSuccess; -} - -hipError_t DynCO::populateDynGlobalFuncs() { - amd::ScopedLock lock(dclock_); - - std::vector func_names; - device::Program* dev_program - = fb_info_->GetProgram(ihipGetDevice())->getDeviceProgram( - *hip::getCurrentDevice()->devices()[0]); - - // Get all the global func names from COMGR - if (!dev_program->getGlobalFuncFromCodeObj(&func_names)) { - LogPrintfError("Could not get Global Funcs from Code Obj for Module: 0x%x \n", module()); - return hipErrorSharedObjectSymbolNotFound; - } - - for (auto& elem : func_names) { - functions_.insert(std::make_pair(elem, new Function(elem))); - } - - return hipSuccess; -} - -//Static Code Object -StatCO::StatCO() { -} - -StatCO::~StatCO() { - amd::ScopedLock lock(sclock_); - - for (auto& elem : functions_) { - delete elem.second; - } - functions_.clear(); - - for (auto& elem : vars_) { - delete elem.second; - } - vars_.clear(); -} - -hipError_t StatCO::digestFatBinary(const void* data, FatBinaryInfo*& programs) { - amd::ScopedLock lock(sclock_); - - if (programs != nullptr) { - return hipSuccess; - } - - // Create a new fat binary object and extract the fat binary for all devices. - programs = new FatBinaryInfo(nullptr, data); - IHIP_RETURN_ONFAIL(programs->ExtractFatBinary(g_devices)); - - return hipSuccess; -} - -FatBinaryInfo** StatCO::addFatBinary(const void* data, bool initialized) { - amd::ScopedLock lock(sclock_); - - if (initialized) { - digestFatBinary(data, modules_[data]); - } - return &modules_[data]; -} - -hipError_t StatCO::removeFatBinary(FatBinaryInfo** module) { - amd::ScopedLock lock(sclock_); - - auto vit = vars_.begin(); - while (vit != vars_.end()) { - if (vit->second->moduleInfo() == module) { - delete vit->second; - vit = vars_.erase(vit); - } else { - ++vit; - } - } - - auto it = managedVars_.begin(); - while (it != managedVars_.end()) { - if ((*it)->moduleInfo() == module) { - ihipFree((*it)->getManagedVarPtr()); - delete *it; - managedVars_.erase(it); - } else { - ++it; - } - } - - auto fit = functions_.begin(); - while (fit != functions_.end()) { - if (fit->second->moduleInfo() == module) { - delete fit->second; - fit = functions_.erase(fit); - } else { - ++fit; - } - } - - auto mit = modules_.begin(); - while (mit != modules_.end()) { - if (&mit->second == module) { - delete mit->second; - mit = modules_.erase(mit); - } else { - ++mit; - } - } - - return hipSuccess; -} - -hipError_t StatCO::registerStatFunction(const void* hostFunction, Function* func) { - amd::ScopedLock lock(sclock_); - - if (functions_.find(hostFunction) != functions_.end()) { - DevLogPrintfError("hostFunctionPtr: 0x%x already exists", hostFunction); - } - functions_.insert(std::make_pair(hostFunction, func)); - - return hipSuccess; -} - -hipError_t StatCO::getStatFunc(hipFunction_t* hfunc, const void* hostFunction, int deviceId) { - amd::ScopedLock lock(sclock_); - - const auto it = functions_.find(hostFunction); - if (it == functions_.end()) { - return hipErrorInvalidSymbol; - } - - return it->second->getStatFunc(hfunc, deviceId); -} - -hipError_t StatCO::getStatFuncAttr(hipFuncAttributes* func_attr, const void* hostFunction, int deviceId) { - amd::ScopedLock lock(sclock_); - - const auto it = functions_.find(hostFunction); - if (it == functions_.end()) { - return hipErrorInvalidSymbol; - } - - return it->second->getStatFuncAttr(func_attr, deviceId); -} - -hipError_t StatCO::registerStatGlobalVar(const void* hostVar, Var* var) { - amd::ScopedLock lock(sclock_); - - if (vars_.find(hostVar) != vars_.end()) { - return hipErrorInvalidSymbol; - } - - vars_.insert(std::make_pair(hostVar, var)); - return hipSuccess; -} - -hipError_t StatCO::getStatGlobalVar(const void* hostVar, int deviceId, hipDeviceptr_t* dev_ptr, - size_t* size_ptr) { - amd::ScopedLock lock(sclock_); - - const auto it = vars_.find(hostVar); - if (it == vars_.end()) { - return hipErrorInvalidSymbol; - } - - DeviceVar* dvar = nullptr; - IHIP_RETURN_ONFAIL(it->second->getStatDeviceVar(&dvar, deviceId)); - - *dev_ptr = dvar->device_ptr(); - *size_ptr = dvar->size(); - return hipSuccess; -} - -hipError_t StatCO::registerStatManagedVar(Var* var) { - managedVars_.emplace_back(var); - return hipSuccess; -} - -hipError_t StatCO::initStatManagedVarDevicePtr(int deviceId) { - amd::ScopedLock lock(sclock_); - - if (managedVarsDevicePtrInitalized_.find(deviceId) == managedVarsDevicePtrInitalized_.end() || - !managedVarsDevicePtrInitalized_[deviceId]) { - for (auto var : managedVars_) { - DeviceVar* dvar = nullptr; - IHIP_RETURN_ONFAIL(var->getStatDeviceVar(&dvar, deviceId)); - - amd::HostQueue* queue = hip::getNullStream(); - if(queue != nullptr) { - ihipMemcpy(reinterpret_cast
(dvar->device_ptr()), var->getManagedVarPtr(), - dvar->size(), hipMemcpyHostToDevice, *queue); - } else { - ClPrint(amd::LOG_ERROR, amd::LOG_API, "Host Queue is NULL"); - return hipErrorInvalidResourceHandle; - } - } - managedVarsDevicePtrInitalized_[deviceId] = true; - } - return hipSuccess; -} -}; //namespace: hip diff --git a/src/hipamd/src/hip_code_object.hpp b/src/hipamd/src/hip_code_object.hpp deleted file mode 100755 index b647c62699..0000000000 --- a/src/hipamd/src/hip_code_object.hpp +++ /dev/null @@ -1,165 +0,0 @@ -/* -Copyright (c) 2015-2020 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef HIP_CODE_OBJECT_HPP -#define HIP_CODE_OBJECT_HPP - -#include "hip_global.hpp" - -#include -#include - -#include "hip/hip_runtime.h" -#include "hip/hip_runtime_api.h" -#include "hip_internal.hpp" -#include "device/device.hpp" -#include "platform/program.hpp" - -//Forward Declaration for friend usage -class PlatformState; - -namespace hip { - -//Code Object base class -class CodeObject { - public: - virtual ~CodeObject() {} - - // Functions to add_dev_prog and build - static hipError_t add_program(int deviceId, hipModule_t hmod, const void* binary_ptr, - size_t binary_size); - static hipError_t build_module(hipModule_t hmod, const std::vector& devices); - - // Given an file desc and file size, extracts to code object for corresponding devices, - // return code_objs{binary_ptr, binary_size}, which could be used to determine foffset - static hipError_t ExtractCodeObjectFromFile(amd::Os::FileDesc fdesc, size_t fsize, - const void ** image, const std::vector& device_names, - std::vector>& code_objs); - - // Given an ptr to memory, extracts to code object for corresponding devices, - // returns code_objs{binary_ptr, binary_size} and uniform resource indicator - static hipError_t ExtractCodeObjectFromMemory(const void* data, - const std::vector& device_names, - std::vector>& code_objs, - std::string& uri); - - static uint64_t ElfSize(const void* emi); - -protected: - //Given an ptr to image or file, extracts to code object - //for corresponding devices - static hipError_t extractCodeObjectFromFatBinary(const void*, - const std::vector&, - std::vector>&); - - CodeObject() {} -private: - friend const std::vector& modules(); -}; - -//Dynamic Code Object -class DynCO : public CodeObject { - amd::Monitor dclock_{"Guards Dynamic Code object", true}; - -public: - DynCO() : device_id_(ihipGetDevice()) {} - virtual ~DynCO(); - - //LoadsCodeObject and its data - hipError_t loadCodeObject(const char* fname, const void* image=nullptr); - hipModule_t module() { return fb_info_->Module(ihipGetDevice()); }; - - //Gets GlobalVar/Functions from a dynamically loaded code object - hipError_t getDynFunc(hipFunction_t* hfunc, std::string func_name); - hipError_t getDeviceVar(DeviceVar** dvar, std::string var_name); - - hipError_t getManagedVarPointer(std::string name, void** pointer, size_t* size_ptr) const { - auto it = vars_.find(name); - if (it != vars_.end() && it->second->getVarKind() == Var::DVK_Managed) { - *pointer = it->second->getManagedVarPtr(); - *size_ptr = it->second->getSize(); - } - return hipSuccess; - } - // Device ID Check to check if module is launched in the same device it was loaded. - inline void CheckDeviceIdMatch() { - if (device_id_ != ihipGetDevice()) { - guarantee(false, "Device mismatch from where this module is loaded"); - } - } - -private: - int device_id_; - FatBinaryInfo* fb_info_; - - //Maps for vars/funcs, could be keyed in with std::string name - std::unordered_map functions_; - std::unordered_map vars_; - - //Populate Global Vars/Funcs from an code object(@ module_load) - hipError_t populateDynGlobalFuncs(); - hipError_t populateDynGlobalVars(); - hipError_t initDynManagedVars(const std::string& managedVar); -}; - -//Static Code Object -class StatCO: public CodeObject { - amd::Monitor sclock_{"Guards Static Code object", true}; -public: - StatCO(); - virtual ~StatCO(); - - //Add/Remove/Digest Fat Binaries passed to us from "__hipRegisterFatBinary" - FatBinaryInfo** addFatBinary(const void* data, bool initialized); - hipError_t removeFatBinary(FatBinaryInfo** module); - hipError_t digestFatBinary(const void* data, FatBinaryInfo*& programs); - - //Register vars/funcs given to use from __hipRegister[Var/Func/ManagedVar] - hipError_t registerStatFunction(const void* hostFunction, Function* func); - hipError_t registerStatGlobalVar(const void* hostVar, Var* var); - hipError_t registerStatManagedVar(Var *var); - - //Retrive Vars/Funcs for a given hostSidePtr(const void*), unless stated otherwise. - hipError_t getStatFunc(hipFunction_t* hfunc, const void* hostFunction, int deviceId); - hipError_t getStatFuncAttr(hipFuncAttributes* func_attr, const void* hostFunction, int deviceId); - hipError_t getStatGlobalVar(const void* hostVar, int deviceId, hipDeviceptr_t* dev_ptr, - size_t* size_ptr); - - //Managed variable is a defined symbol in code object - //pointer to the alocated managed memory has to be copied to the address of symbol - hipError_t initStatManagedVarDevicePtr(int deviceId); -private: - friend class ::PlatformState; - //Populated during __hipRegisterFatBinary - std::unordered_map modules_; - //Populated during __hipRegisterFuncs - std::unordered_map functions_; - //Populated during __hipRegisterVars - std::unordered_map vars_; - //Populated during __hipRegisterManagedVar - std::vector managedVars_; - std::unordered_map managedVarsDevicePtrInitalized_; -}; - -}; // namespace hip - -#endif /* HIP_CODE_OBJECT_HPP */ diff --git a/src/hipamd/src/hip_context.cpp b/src/hipamd/src/hip_context.cpp deleted file mode 100755 index 60bb43421d..0000000000 --- a/src/hipamd/src/hip_context.cpp +++ /dev/null @@ -1,377 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include "hip_internal.hpp" -#include "hip_platform.hpp" -#include "platform/runtime.hpp" -#include "utils/flags.hpp" -#include "utils/versions.hpp" - -std::vector g_devices; - -namespace hip { - -thread_local Device* g_device = nullptr; -thread_local std::stack g_ctxtStack; -thread_local hipError_t g_lastError = hipSuccess; -std::once_flag g_ihipInitialized; -Device* host_device = nullptr; - -void init() { - if (!amd::Runtime::initialized()) { - amd::IS_HIP = true; - GPU_NUM_MEM_DEPENDENCY = 0; - AMD_DIRECT_DISPATCH = flagIsDefault(AMD_DIRECT_DISPATCH) ? false : AMD_DIRECT_DISPATCH; - amd::Runtime::init(); - } - - const std::vector& devices = amd::Device::getDevices(CL_DEVICE_TYPE_GPU, false); - - for (unsigned int i=0; i device(1, devices[i]); - amd::Context* context = new amd::Context(device, amd::Context::Info()); - if (!context) return; - - // Enable active wait on the device by default - devices[i]->SetActiveWait(true); - - if (context && CL_SUCCESS != context->create(nullptr)) { - context->release(); - } else { - g_devices.push_back(new Device(context, i)); - } - } - - amd::Context* hContext = new amd::Context(devices, amd::Context::Info()); - if (!hContext) return; - - if (CL_SUCCESS != hContext->create(nullptr)) { - hContext->release(); - } - host_device = new Device(hContext, -1); - - PlatformState::instance().init(); -} - -Device* getCurrentDevice() { - return g_device; -} - -void setCurrentDevice(unsigned int index) { - assert(index(stream)->asHostQueue(); - if (!(reinterpret_cast(stream)->Flags() & hipStreamNonBlocking)) { - iHipWaitActiveStreams(queue, WaitNullStreamOnly); - } - return queue; - } -} - -// ================================================================================================ -amd::HostQueue* getNullStream(amd::Context& ctx) { - for (auto& it : g_devices) { - if (it->asContext() == &ctx) { - return it->NullStream(); - } - } - // If it's a pure SVM allocation with system memory access, then it shouldn't matter which device - // runtime selects by default - if (hip::host_device->asContext() == &ctx) { - // Return current... - return getNullStream(); - } - return nullptr; -} - -// ================================================================================================ -amd::HostQueue* getNullStream() { - Device* device = getCurrentDevice(); - return device ? device->NullStream() : nullptr; -} - -}; - -using namespace hip; - -hipError_t hipInit(unsigned int flags) { - HIP_INIT_API(hipInit, flags); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxCreate(hipCtx_t *ctx, unsigned int flags, hipDevice_t device) { - HIP_INIT_API(hipCtxCreate, ctx, flags, device); - - if (static_cast(device) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidValue); - } - - *ctx = reinterpret_cast(g_devices[device]); - - // Increment ref count for device primary context - g_devices[device]->retain(); - g_ctxtStack.push(g_devices[device]); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxSetCurrent(hipCtx_t ctx) { - HIP_INIT_API(hipCtxSetCurrent, ctx); - - if (ctx == nullptr) { - if(!g_ctxtStack.empty()) { - g_ctxtStack.pop(); - } - } else { - hip::g_device = reinterpret_cast(ctx); - if(!g_ctxtStack.empty()) { - g_ctxtStack.pop(); - } - g_ctxtStack.push(hip::getCurrentDevice()); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxGetCurrent(hipCtx_t* ctx) { - HIP_INIT_API(hipCtxGetCurrent, ctx); - - *ctx = reinterpret_cast(hip::getCurrentDevice()); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxGetSharedMemConfig(hipSharedMemConfig* pConfig) { - HIP_INIT_API(hipCtxGetSharedMemConfig, pConfig); - - *pConfig = hipSharedMemBankSizeFourByte; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipRuntimeGetVersion(int *runtimeVersion) { - HIP_INIT_API(hipRuntimeGetVersion, runtimeVersion); - - if (!runtimeVersion) { - HIP_RETURN(hipErrorInvalidValue); - } - - // HIP_VERSION = HIP_VERSION_MAJOR*100 + HIP_MINOR_VERSION - *runtimeVersion = HIP_VERSION; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxDestroy(hipCtx_t ctx) { - HIP_INIT_API(hipCtxDestroy, ctx); - - hip::Device* dev = reinterpret_cast(ctx); - if (dev == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - // Need to remove the ctx of calling thread if its the top one - if (!g_ctxtStack.empty() && g_ctxtStack.top() == dev) { - g_ctxtStack.pop(); - } - - // Remove context from global context list - for (unsigned int i = 0; i < g_devices.size(); i++) { - if (g_devices[i] == dev) { - // Decrement ref count for device primary context - dev->release(); - } - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxPopCurrent(hipCtx_t* ctx) { - HIP_INIT_API(hipCtxPopCurrent, ctx); - - hip::Device** dev = reinterpret_cast(ctx); - if (!g_ctxtStack.empty()) { - if (dev != nullptr) { - *dev = g_ctxtStack.top(); - } - g_ctxtStack.pop(); - } else { - DevLogError("Context Stack empty \n"); - HIP_RETURN(hipErrorInvalidContext); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxPushCurrent(hipCtx_t ctx) { - HIP_INIT_API(hipCtxPushCurrent, ctx); - - hip::Device* dev = reinterpret_cast(ctx); - if (dev == nullptr) { - HIP_RETURN(hipErrorInvalidContext); - } - - hip::g_device = dev; - g_ctxtStack.push(hip::getCurrentDevice()); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDriverGetVersion(int* driverVersion) { - HIP_INIT_API(hipDriverGetVersion, driverVersion); - - if (!driverVersion) { - HIP_RETURN(hipErrorInvalidValue); - } - - // HIP_VERSION = HIP_VERSION_MAJOR*100 + HIP_MINOR_VERSION - *driverVersion = HIP_VERSION; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxGetDevice(hipDevice_t* device) { - HIP_INIT_API(hipCtxGetDevice, device); - - if (device != nullptr) { - *device = hip::getCurrentDevice()->deviceId(); - HIP_RETURN(hipSuccess); - } else { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipErrorInvalidContext); -} - -hipError_t hipCtxGetApiVersion(hipCtx_t ctx, int* apiVersion) { - HIP_INIT_API(hipCtxGetApiVersion, apiVersion); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipCtxGetCacheConfig(hipFuncCache_t* cacheConfig) { - HIP_INIT_API(hipCtxGetCacheConfig, cacheConfig); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipCtxSetCacheConfig(hipFuncCache_t cacheConfig) { - HIP_INIT_API(hipCtxSetCacheConfig, cacheConfig); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipCtxSetSharedMemConfig(hipSharedMemConfig config) { - HIP_INIT_API(hipCtxSetSharedMemConfig, config); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipCtxSynchronize(void) { - HIP_INIT_API(hipCtxSynchronize, 1); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipCtxGetFlags(unsigned int* flags) { - HIP_INIT_API(hipCtxGetFlags, flags); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipDevicePrimaryCtxGetState(hipDevice_t dev, unsigned int* flags, int* active) { - HIP_INIT_API(hipDevicePrimaryCtxGetState, dev, flags, active); - - if (static_cast(dev) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidDevice); - } - - if (flags != nullptr) { - *flags = 0; - } - - if (active != nullptr) { - *active = (g_devices[dev] == hip::getCurrentDevice())? 1 : 0; - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDevicePrimaryCtxRelease(hipDevice_t dev) { - HIP_INIT_API(hipDevicePrimaryCtxRelease, dev); - - if (static_cast(dev) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidDevice); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDevicePrimaryCtxRetain(hipCtx_t* pctx, hipDevice_t dev) { - HIP_INIT_API(hipDevicePrimaryCtxRetain, pctx, dev); - - if (static_cast(dev) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidDevice); - } - if (pctx == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pctx = reinterpret_cast(g_devices[dev]); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDevicePrimaryCtxReset(hipDevice_t dev) { - HIP_INIT_API(hipDevicePrimaryCtxReset, dev); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDevicePrimaryCtxSetFlags(hipDevice_t dev, unsigned int flags) { - HIP_INIT_API(hipDevicePrimaryCtxSetFlags, dev, flags); - - if (static_cast(dev) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidDevice); - } else { - HIP_RETURN(hipErrorContextAlreadyInUse); - } -} diff --git a/src/hipamd/src/hip_conversions.hpp b/src/hipamd/src/hip_conversions.hpp deleted file mode 100644 index 03d6320b60..0000000000 --- a/src/hipamd/src/hip_conversions.hpp +++ /dev/null @@ -1,903 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#pragma once - -#include -#include - -namespace hip -{ -inline -cl_channel_type getCLChannelType(const hipArray_Format hipFormat, - const hipTextureReadMode hipReadMode) { - if (hipReadMode == hipReadModeElementType) { - switch (hipFormat) { - case HIP_AD_FORMAT_UNSIGNED_INT8: - return CL_UNSIGNED_INT8; - case HIP_AD_FORMAT_SIGNED_INT8: - return CL_SIGNED_INT8; - case HIP_AD_FORMAT_UNSIGNED_INT16: - return CL_UNSIGNED_INT16; - case HIP_AD_FORMAT_SIGNED_INT16: - return CL_SIGNED_INT16; - case HIP_AD_FORMAT_UNSIGNED_INT32: - return CL_UNSIGNED_INT32; - case HIP_AD_FORMAT_SIGNED_INT32: - return CL_SIGNED_INT32; - case HIP_AD_FORMAT_HALF: - return CL_HALF_FLOAT; - case HIP_AD_FORMAT_FLOAT: - return CL_FLOAT; - } - } else if (hipReadMode == hipReadModeNormalizedFloat) { - switch (hipFormat) { - case HIP_AD_FORMAT_UNSIGNED_INT8: - return CL_UNORM_INT8; - case HIP_AD_FORMAT_SIGNED_INT8: - return CL_SNORM_INT8; - case HIP_AD_FORMAT_UNSIGNED_INT16: - return CL_UNORM_INT16; - case HIP_AD_FORMAT_SIGNED_INT16: - return CL_SNORM_INT16; - case HIP_AD_FORMAT_UNSIGNED_INT32: - return CL_UNSIGNED_INT32; - case HIP_AD_FORMAT_SIGNED_INT32: - return CL_SIGNED_INT32; - case HIP_AD_FORMAT_HALF: - return CL_HALF_FLOAT; - case HIP_AD_FORMAT_FLOAT: - return CL_FLOAT; - } - } - - ShouldNotReachHere(); - - return {}; -} - -inline -cl_channel_order getCLChannelOrder(const unsigned int hipNumChannels, - const int sRGB) { - switch (hipNumChannels) { - case 1: - return CL_R; - case 2: - return CL_RG; - case 4: - return (sRGB == 1) ? CL_sRGBA : CL_RGBA; - default: - break; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -cl_mem_object_type getCLMemObjectType(const unsigned int hipWidth, - const unsigned int hipHeight, - const unsigned int hipDepth, - const unsigned int flags) { - if (flags == hipArrayDefault) { - if ((hipWidth != 0) && (hipHeight == 0) && (hipDepth == 0)) { - return CL_MEM_OBJECT_IMAGE1D; - } else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth == 0)) { - return CL_MEM_OBJECT_IMAGE2D; - } else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth != 0)) { - return CL_MEM_OBJECT_IMAGE3D; - } - } else if (flags == hipArrayLayered) { - if ((hipWidth != 0) && (hipHeight == 0) && (hipDepth != 0)) { - return CL_MEM_OBJECT_IMAGE1D_ARRAY; - } else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth != 0)) { - return CL_MEM_OBJECT_IMAGE2D_ARRAY; - } - } - - ShouldNotReachHere(); - - return {}; -} - -inline -cl_addressing_mode getCLAddressingMode(const hipTextureAddressMode hipAddressMode) { - switch (hipAddressMode) { - case hipAddressModeWrap: - return CL_ADDRESS_REPEAT; - case hipAddressModeClamp: - return CL_ADDRESS_CLAMP_TO_EDGE; - case hipAddressModeMirror: - return CL_ADDRESS_MIRRORED_REPEAT; - case hipAddressModeBorder: - return CL_ADDRESS_CLAMP; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -cl_filter_mode getCLFilterMode(const hipTextureFilterMode hipFilterMode) { - switch (hipFilterMode) { - case hipFilterModePoint: - return CL_FILTER_NEAREST; - case hipFilterModeLinear: - return CL_FILTER_LINEAR; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -cl_mem_object_type getCLMemObjectType(const hipResourceType hipResType) { - switch (hipResType) { - case hipResourceTypeLinear: - return CL_MEM_OBJECT_IMAGE1D_BUFFER; - case hipResourceTypePitch2D: - return CL_MEM_OBJECT_IMAGE2D; - default: - break; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -size_t getElementSize(const hipArray_const_t array) { - switch (array->Format) { - case HIP_AD_FORMAT_UNSIGNED_INT8: - case HIP_AD_FORMAT_SIGNED_INT8: - return 1 * array->NumChannels; - case HIP_AD_FORMAT_UNSIGNED_INT16: - case HIP_AD_FORMAT_SIGNED_INT16: - case HIP_AD_FORMAT_HALF: - return 2 * array->NumChannels; - case HIP_AD_FORMAT_UNSIGNED_INT32: - case HIP_AD_FORMAT_SIGNED_INT32: - case HIP_AD_FORMAT_FLOAT: - return 4 * array->NumChannels; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -hipChannelFormatDesc getChannelFormatDesc(int numChannels, - hipArray_Format arrayFormat) { - switch (arrayFormat) { - case HIP_AD_FORMAT_UNSIGNED_INT8: - switch (numChannels) { - case 1: - return {8, 0, 0, 0, hipChannelFormatKindUnsigned}; - case 2: - return {8, 8, 0, 0, hipChannelFormatKindUnsigned}; - case 4: - return {8, 8, 8, 8, hipChannelFormatKindUnsigned}; - } - case HIP_AD_FORMAT_SIGNED_INT8: - switch (numChannels) { - case 1: - return {8, 0, 0, 0, hipChannelFormatKindSigned}; - case 2: - return {8, 8, 0, 0, hipChannelFormatKindSigned}; - case 4: - return {8, 8, 8, 8, hipChannelFormatKindSigned}; - } - case HIP_AD_FORMAT_UNSIGNED_INT16: - switch (numChannels) { - case 1: - return {16, 0, 0, 0, hipChannelFormatKindUnsigned}; - case 2: - return {16, 16, 0, 0, hipChannelFormatKindUnsigned}; - case 4: - return {16, 16, 16, 16, hipChannelFormatKindUnsigned}; - } - case HIP_AD_FORMAT_SIGNED_INT16: - switch (numChannels) { - case 1: - return {16, 0, 0, 0, hipChannelFormatKindSigned}; - case 2: - return {16, 16, 0, 0, hipChannelFormatKindSigned}; - case 4: - return {16, 16, 16, 16, hipChannelFormatKindSigned}; - } - case HIP_AD_FORMAT_UNSIGNED_INT32: - switch (numChannels) { - case 1: - return {32, 0, 0, 0, hipChannelFormatKindUnsigned}; - case 2: - return {32, 32, 0, 0, hipChannelFormatKindUnsigned}; - case 4: - return {32, 32, 32, 32, hipChannelFormatKindUnsigned}; - } - case HIP_AD_FORMAT_SIGNED_INT32: - switch (numChannels) { - case 1: - return {32, 0, 0, 0, hipChannelFormatKindSigned}; - case 2: - return {32, 32, 0, 0, hipChannelFormatKindSigned}; - case 4: - return {32, 32, 32, 32, hipChannelFormatKindSigned}; - } - case HIP_AD_FORMAT_HALF: - switch (numChannels) { - case 1: - return {16, 0, 0, 0, hipChannelFormatKindFloat}; - case 2: - return {16, 16, 0, 0, hipChannelFormatKindFloat}; - case 4: - return {16, 16, 16, 16, hipChannelFormatKindFloat}; - } - case HIP_AD_FORMAT_FLOAT: - switch (numChannels) { - case 1: - return {32, 0, 0, 0, hipChannelFormatKindFloat}; - case 2: - return {32, 32, 0, 0, hipChannelFormatKindFloat}; - case 4: - return {32, 32, 32, 32, hipChannelFormatKindFloat}; - } - } - - ShouldNotReachHere(); - - return {}; -} - -inline -unsigned int getNumChannels(const hipChannelFormatDesc& desc) { - return ((desc.x != 0) + (desc.y != 0) + (desc.z != 0) + (desc.w != 0)); -} - -inline -hipArray_Format getArrayFormat(const hipChannelFormatDesc& desc) { - switch (desc.f) { - case hipChannelFormatKindUnsigned: - switch (desc.x) { - case 8: - return HIP_AD_FORMAT_UNSIGNED_INT8; - case 16: - return HIP_AD_FORMAT_UNSIGNED_INT16; - case 32: - return HIP_AD_FORMAT_UNSIGNED_INT32; - } - case hipChannelFormatKindSigned: - switch (desc.x) { - case 8: - return HIP_AD_FORMAT_SIGNED_INT8; - case 16: - return HIP_AD_FORMAT_SIGNED_INT16; - case 32: - return HIP_AD_FORMAT_SIGNED_INT32; - } - case hipChannelFormatKindFloat: - switch (desc.x) { - case 16: - return HIP_AD_FORMAT_HALF; - case 32: - return HIP_AD_FORMAT_FLOAT; - } - default: - break; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -int getNumChannels(const hipResourceViewFormat hipFormat) { - switch (hipFormat) { - case hipResViewFormatUnsignedChar1: - case hipResViewFormatSignedChar1: - case hipResViewFormatUnsignedShort1: - case hipResViewFormatSignedShort1: - case hipResViewFormatUnsignedInt1: - case hipResViewFormatSignedInt1: - case hipResViewFormatHalf1: - case hipResViewFormatFloat1: - return 1; - case hipResViewFormatUnsignedChar2: - case hipResViewFormatSignedChar2: - case hipResViewFormatUnsignedShort2: - case hipResViewFormatSignedShort2: - case hipResViewFormatUnsignedInt2: - case hipResViewFormatSignedInt2: - case hipResViewFormatHalf2: - case hipResViewFormatFloat2: - return 2; - case hipResViewFormatUnsignedChar4: - case hipResViewFormatSignedChar4: - case hipResViewFormatUnsignedShort4: - case hipResViewFormatSignedShort4: - case hipResViewFormatUnsignedInt4: - case hipResViewFormatSignedInt4: - case hipResViewFormatHalf4: - case hipResViewFormatFloat4: - return 4; - default: - break; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -hipArray_Format getArrayFormat(const hipResourceViewFormat hipFormat) { - switch (hipFormat) { - case hipResViewFormatUnsignedChar1: - case hipResViewFormatUnsignedChar2: - case hipResViewFormatUnsignedChar4: - return HIP_AD_FORMAT_UNSIGNED_INT8; - case hipResViewFormatSignedChar1: - case hipResViewFormatSignedChar2: - case hipResViewFormatSignedChar4: - return HIP_AD_FORMAT_SIGNED_INT8; - case hipResViewFormatUnsignedShort1: - case hipResViewFormatUnsignedShort2: - case hipResViewFormatUnsignedShort4: - return HIP_AD_FORMAT_UNSIGNED_INT16; - case hipResViewFormatSignedShort1: - case hipResViewFormatSignedShort2: - case hipResViewFormatSignedShort4: - return HIP_AD_FORMAT_SIGNED_INT16; - case hipResViewFormatUnsignedInt1: - case hipResViewFormatUnsignedInt2: - case hipResViewFormatUnsignedInt4: - return HIP_AD_FORMAT_UNSIGNED_INT32; - case hipResViewFormatSignedInt1: - case hipResViewFormatSignedInt2: - case hipResViewFormatSignedInt4: - return HIP_AD_FORMAT_SIGNED_INT32; - case hipResViewFormatHalf1: - case hipResViewFormatHalf2: - case hipResViewFormatHalf4: - return HIP_AD_FORMAT_HALF; - case hipResViewFormatFloat1: - case hipResViewFormatFloat2: - case hipResViewFormatFloat4: - return HIP_AD_FORMAT_FLOAT; - default: - break; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -hipResourceViewFormat getResourceViewFormat(const hipChannelFormatDesc& desc) { - switch (desc.f) { - case hipChannelFormatKindUnsigned: - switch (getNumChannels(desc)) { - case 1: - switch (desc.x) { - case 8: - return hipResViewFormatUnsignedChar1; - case 16: - return hipResViewFormatUnsignedShort1; - case 32: - return hipResViewFormatUnsignedInt1; - } - case 2: - switch (desc.x) { - case 8: - return hipResViewFormatUnsignedChar2; - case 16: - return hipResViewFormatUnsignedShort2; - case 32: - return hipResViewFormatUnsignedInt2; - } - case 4: - switch (desc.x) { - case 8: - return hipResViewFormatUnsignedChar4; - case 16: - return hipResViewFormatUnsignedShort4; - case 32: - return hipResViewFormatUnsignedInt4; - } - } - case hipChannelFormatKindSigned: - switch (getNumChannels(desc)) { - case 1: - switch (desc.x) { - case 8: - return hipResViewFormatSignedChar1; - case 16: - return hipResViewFormatSignedShort1; - case 32: - return hipResViewFormatSignedInt1; - } - case 2: - switch (desc.x) { - case 8: - return hipResViewFormatSignedChar2; - case 16: - return hipResViewFormatSignedShort2; - case 32: - return hipResViewFormatSignedInt2; - } - case 4: - switch (desc.x) { - case 8: - return hipResViewFormatSignedChar4; - case 16: - return hipResViewFormatSignedShort4; - case 32: - return hipResViewFormatSignedInt4; - } - } - case hipChannelFormatKindFloat: - switch (getNumChannels(desc)) { - case 1: - switch (desc.x) { - case 16: - return hipResViewFormatHalf1; - case 32: - return hipResViewFormatFloat1; - } - case 2: - switch (desc.x) { - case 16: - return hipResViewFormatHalf2; - case 32: - return hipResViewFormatFloat2; - } - case 4: - switch (desc.x) { - case 16: - return hipResViewFormatHalf4; - case 32: - return hipResViewFormatFloat4; - } - } - default: - break; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -hipTextureDesc getTextureDesc(const textureReference* texRef) { - hipTextureDesc texDesc = {}; - std::memcpy(texDesc.addressMode, texRef->addressMode, sizeof(texDesc.addressMode)); - texDesc.filterMode = texRef->filterMode; - texDesc.readMode = texRef->readMode; - texDesc.sRGB = texRef->sRGB; - texDesc.normalizedCoords = texRef->normalized; - texDesc.maxAnisotropy = texRef->maxAnisotropy; - texDesc.mipmapFilterMode = texRef->mipmapFilterMode; - texDesc.mipmapLevelBias = texRef->mipmapLevelBias; - texDesc.minMipmapLevelClamp = texRef->minMipmapLevelClamp; - texDesc.maxMipmapLevelClamp = texRef->maxMipmapLevelClamp; - - return texDesc; -} - -inline -hipResourceViewDesc getResourceViewDesc(hipArray_const_t array, - const hipResourceViewFormat format) { - hipResourceViewDesc resViewDesc = {}; - resViewDesc.format = format; - resViewDesc.width = array->width; - resViewDesc.height = array->height; - resViewDesc.depth = array->depth; - resViewDesc.firstMipmapLevel = 0; - resViewDesc.lastMipmapLevel = 0; - resViewDesc.firstLayer = 0; - resViewDesc.lastLayer = 0; /* TODO add hipArray::numLayers */ - - return resViewDesc; -} - -inline -hipResourceViewDesc getResourceViewDesc(hipMipmappedArray_const_t array, - const hipResourceViewFormat format) { - hipResourceViewDesc resViewDesc = {}; - resViewDesc.format = format; - resViewDesc.width = array->width; - resViewDesc.height = array->height; - resViewDesc.depth = array->depth; - resViewDesc.firstMipmapLevel = 0; - resViewDesc.lastMipmapLevel = 0; /* TODO add hipMipmappedArray::numMipLevels */ - resViewDesc.firstLayer = 0; - resViewDesc.lastLayer = 0; /* TODO add hipArray::numLayers */ - - return resViewDesc; -} - -inline -std::pair getMemoryType(const hipMemcpyKind kind) { - switch (kind) { - case hipMemcpyHostToHost: - return {hipMemoryTypeHost, hipMemoryTypeHost}; - case hipMemcpyHostToDevice: - return {hipMemoryTypeHost, hipMemoryTypeDevice}; - case hipMemcpyDeviceToHost: - return {hipMemoryTypeDevice, hipMemoryTypeHost}; - case hipMemcpyDeviceToDevice: - return {hipMemoryTypeDevice, hipMemoryTypeDevice}; - case hipMemcpyDefault: - return {hipMemoryTypeUnified, hipMemoryTypeUnified}; - } - - ShouldNotReachHere(); - - return {}; -} - -inline -HIP_MEMCPY3D getDrvMemcpy3DDesc(const hip_Memcpy2D& desc2D) { - HIP_MEMCPY3D desc3D = {}; - - desc3D.srcXInBytes = desc2D.srcXInBytes; - desc3D.srcY = desc2D.srcY; - desc3D.srcZ = 0; - desc3D.srcLOD = 0; - desc3D.srcMemoryType = desc2D.srcMemoryType; - desc3D.srcHost = desc2D.srcHost; - desc3D.srcDevice = desc2D.srcDevice; - desc3D.srcArray = desc2D.srcArray; - desc3D.srcPitch = desc2D.srcPitch; - desc3D.srcHeight = 0; - - desc3D.dstXInBytes = desc2D.dstXInBytes; - desc3D.dstY = desc2D.dstY; - desc3D.dstZ = 0; - desc3D.dstLOD = 0; - desc3D.dstMemoryType = desc2D.dstMemoryType; - desc3D.dstHost = desc2D.dstHost; - desc3D.dstDevice = desc2D.dstDevice; - desc3D.dstArray = desc2D.dstArray; - desc3D.dstPitch = desc2D.dstPitch; - desc3D.dstHeight = 0; - - desc3D.WidthInBytes = desc2D.WidthInBytes; - desc3D.Height = desc2D.Height; - desc3D.Depth = 1; - - return desc3D; -} - -inline -HIP_MEMCPY3D getDrvMemcpy3DDesc(const hipMemcpy3DParms& desc) { - HIP_MEMCPY3D descDrv = {}; - - descDrv.WidthInBytes = desc.extent.width; - descDrv.Height = desc.extent.height; - descDrv.Depth = desc.extent.depth; - - descDrv.srcXInBytes = desc.srcPos.x; - descDrv.srcY = desc.srcPos.y; - descDrv.srcZ = desc.srcPos.z; - descDrv.srcLOD = 0; - - descDrv.dstXInBytes = desc.dstPos.x; - descDrv.dstY = desc.dstPos.y; - descDrv.dstZ = desc.dstPos.z; - descDrv.dstLOD = 0; - - if (desc.srcArray != nullptr) { - descDrv.srcMemoryType = hipMemoryTypeArray; - descDrv.srcArray = desc.srcArray; - // When reffering to array memory, hipPos::x is in elements. - descDrv.srcXInBytes *= getElementSize(desc.srcArray); - } - - if (desc.srcPtr.ptr != nullptr) { - descDrv.srcMemoryType = std::get<0>(hip::getMemoryType(desc.kind)); - descDrv.srcHost = desc.srcPtr.ptr; - descDrv.srcDevice = desc.srcPtr.ptr; - descDrv.srcPitch = desc.srcPtr.pitch; - descDrv.srcHeight = desc.srcPtr.ysize; - } - - if (desc.dstArray != nullptr) { - descDrv.dstMemoryType = hipMemoryTypeArray; - descDrv.dstArray = desc.dstArray; - // When reffering to array memory, hipPos::x is in elements. - descDrv.dstXInBytes *= getElementSize(desc.dstArray); - } - - if (desc.dstPtr.ptr != nullptr) { - descDrv.dstMemoryType = std::get<1>(getMemoryType(desc.kind)); - descDrv.dstHost = desc.dstPtr.ptr; - descDrv.dstDevice = desc.dstPtr.ptr; - descDrv.dstPitch = desc.dstPtr.pitch; - descDrv.dstHeight = desc.dstPtr.ysize; - } - - // If a HIP array is participating in the copy, the extent is defined in terms of that array's elements. - if ((desc.srcArray != nullptr) && (desc.dstArray == nullptr)) { - descDrv.WidthInBytes *= getElementSize(desc.srcArray); - } else if ((desc.srcArray == nullptr) && (desc.dstArray != nullptr)) { - descDrv.WidthInBytes *= getElementSize(desc.dstArray); - } else if ((desc.srcArray != nullptr) && (desc.dstArray != nullptr)) { - descDrv.WidthInBytes *= getElementSize(desc.dstArray); - } - - return descDrv; -} - -inline -hipResourceType getResourceType(const HIPresourcetype resType) { - // These two enums should be isomorphic. - return static_cast(resType); -} - -inline -HIPresourcetype getResourceType(const hipResourceType resType) { - // These two enums should be isomorphic. - return static_cast(resType); -} - -inline -hipResourceDesc getResourceDesc(const HIP_RESOURCE_DESC& resDesc) { - hipResourceDesc desc; - - desc.resType = getResourceType(resDesc.resType); - switch (desc.resType) { - case hipResourceTypeArray: - desc.res.array.array = resDesc.res.array.hArray; - break; - case hipResourceTypeMipmappedArray: - desc.res.mipmap.mipmap = resDesc.res.mipmap.hMipmappedArray; - break; - case hipResourceTypeLinear: - desc.res.linear.devPtr = resDesc.res.linear.devPtr; - desc.res.linear.desc = getChannelFormatDesc(resDesc.res.linear.numChannels, resDesc.res.linear.format); - desc.res.linear.sizeInBytes = resDesc.res.linear.sizeInBytes; - break; - case hipResourceTypePitch2D: - desc.res.pitch2D.devPtr = resDesc.res.pitch2D.devPtr; - desc.res.pitch2D.desc = getChannelFormatDesc(resDesc.res.pitch2D.numChannels, resDesc.res.pitch2D.format); - desc.res.pitch2D.width = resDesc.res.pitch2D.width; - desc.res.pitch2D.height = resDesc.res.pitch2D.height; - desc.res.pitch2D.pitchInBytes = resDesc.res.pitch2D.pitchInBytes; - break; - default: - break; - } - - return desc; -} - -inline -HIP_RESOURCE_DESC getResourceDesc(const hipResourceDesc& resDesc) { - HIP_RESOURCE_DESC desc; - - desc.resType = getResourceType(resDesc.resType); - switch (desc.resType) { - case HIP_RESOURCE_TYPE_ARRAY: - desc.res.array.hArray = resDesc.res.array.array; - break; - case HIP_RESOURCE_TYPE_MIPMAPPED_ARRAY: - desc.res.mipmap.hMipmappedArray = resDesc.res.mipmap.mipmap; - break; - case HIP_RESOURCE_TYPE_LINEAR: - desc.res.linear.devPtr = resDesc.res.linear.devPtr; - desc.res.linear.numChannels = getNumChannels(resDesc.res.linear.desc); - desc.res.linear.format = getArrayFormat(resDesc.res.linear.desc); - desc.res.linear.sizeInBytes = resDesc.res.linear.sizeInBytes; - break; - case HIP_RESOURCE_TYPE_PITCH2D: - desc.res.pitch2D.devPtr = resDesc.res.pitch2D.devPtr; - desc.res.pitch2D.numChannels = getNumChannels(resDesc.res.pitch2D.desc); - desc.res.pitch2D.format = getArrayFormat(resDesc.res.pitch2D.desc); - desc.res.pitch2D.width = resDesc.res.pitch2D.width; - desc.res.pitch2D.height = resDesc.res.pitch2D.height; - desc.res.pitch2D.pitchInBytes = resDesc.res.pitch2D.pitchInBytes; - break; - default: - break; - } - - return desc; -} - -inline -hipTextureAddressMode getAddressMode(const HIPaddress_mode mode) { - // These two enums should be isomorphic. - return static_cast(mode); -} - -inline -HIPaddress_mode getAddressMode(const hipTextureAddressMode mode) { - // These two enums should be isomorphic. - return static_cast(mode); -} - -inline -hipTextureFilterMode getFilterMode(const HIPfilter_mode mode) { - // These two enums should be isomorphic. - return static_cast(mode); -} - -inline -HIPfilter_mode getFilterMode(const hipTextureFilterMode mode) { - // These two enums should be isomorphic. - return static_cast(mode); -} - -inline -hipTextureReadMode getReadMode(const unsigned int flags) { - if (flags & HIP_TRSF_READ_AS_INTEGER) { - return hipReadModeElementType; - } else { - return hipReadModeNormalizedFloat; - } -} - -inline -unsigned int getReadMode(const hipTextureReadMode mode) { - if (mode == hipReadModeElementType) { - return HIP_TRSF_READ_AS_INTEGER; - } else { - return 0; - } -} - -inline -int getsRGB(const unsigned int flags) { - if (flags & HIP_TRSF_SRGB) { - return 1; - } else { - return 0; - } -} - -inline -unsigned int getsRGB(const int sRGB) { - if (sRGB == 1) { - return HIP_TRSF_SRGB; - } else { - return 0; - } -} - -inline -int getNormalizedCoords(const unsigned int flags) { - if (flags & HIP_TRSF_NORMALIZED_COORDINATES) { - return 1; - } else { - return 0; - } -} - -inline -unsigned int getNormalizedCoords(const int normalizedCoords) { - if (normalizedCoords == 1) { - return HIP_TRSF_NORMALIZED_COORDINATES; - } else { - return 0; - } -} - -inline -hipTextureDesc getTextureDesc(const HIP_TEXTURE_DESC& texDesc) { - hipTextureDesc desc; - - desc.addressMode[0] = getAddressMode(texDesc.addressMode[0]); - desc.addressMode[1] = getAddressMode(texDesc.addressMode[1]); - desc.addressMode[2] = getAddressMode(texDesc.addressMode[2]); - desc.filterMode = getFilterMode(texDesc.filterMode); - desc.readMode = getReadMode(texDesc.flags); - desc.sRGB = getsRGB(texDesc.flags); - std::memcpy(desc.borderColor, texDesc.borderColor, sizeof(desc.borderColor)); - desc.normalizedCoords = getNormalizedCoords(texDesc.flags); - desc.maxAnisotropy = texDesc.maxAnisotropy; - desc.mipmapFilterMode = getFilterMode(texDesc.mipmapFilterMode); - desc.mipmapLevelBias = texDesc.mipmapLevelBias; - desc.minMipmapLevelClamp = texDesc.minMipmapLevelClamp; - desc.maxMipmapLevelClamp = texDesc.maxMipmapLevelClamp; - - return desc; -} - -inline -HIP_TEXTURE_DESC getTextureDesc(const hipTextureDesc& texDesc) { - HIP_TEXTURE_DESC desc; - - desc.addressMode[0] = getAddressMode(texDesc.addressMode[0]); - desc.addressMode[1] = getAddressMode(texDesc.addressMode[1]); - desc.addressMode[2] = getAddressMode(texDesc.addressMode[2]); - desc.filterMode = getFilterMode(texDesc.filterMode); - desc.flags = 0; - desc.flags |= getReadMode(texDesc.readMode); - desc.flags |= getsRGB(texDesc.sRGB); - desc.flags |= getNormalizedCoords(texDesc.normalizedCoords); - desc.maxAnisotropy = texDesc.maxAnisotropy; - desc.mipmapFilterMode = getFilterMode(texDesc.mipmapFilterMode); - desc.mipmapLevelBias = texDesc.mipmapLevelBias; - desc.minMipmapLevelClamp = texDesc.minMipmapLevelClamp; - desc.maxMipmapLevelClamp = texDesc.maxMipmapLevelClamp; - std::memcpy(desc.borderColor, texDesc.borderColor, sizeof(desc.borderColor)); - - return desc; -} - -inline -hipResourceViewFormat getResourceViewFormat(const HIPresourceViewFormat format) { - // These two enums should be isomorphic. - return static_cast(format); -} - -inline -HIPresourceViewFormat getResourceViewFormat(const hipResourceViewFormat format) { - // These two enums should be isomorphic. - return static_cast(format); -} - -inline -hipResourceViewDesc getResourceViewDesc(const HIP_RESOURCE_VIEW_DESC& resViewDesc) { - hipResourceViewDesc desc; - - desc.format = getResourceViewFormat(resViewDesc.format); - desc.width = resViewDesc.width; - desc.height = resViewDesc.height; - desc.depth = resViewDesc.depth; - desc.firstMipmapLevel = resViewDesc.firstMipmapLevel; - desc.lastMipmapLevel = resViewDesc.lastMipmapLevel; - desc.firstLayer = resViewDesc.firstLayer; - desc.lastLayer = resViewDesc.lastLayer; - - return desc; -} - -inline -HIP_RESOURCE_VIEW_DESC getResourceViewDesc(const hipResourceViewDesc& resViewDesc) { - HIP_RESOURCE_VIEW_DESC desc; - - desc.format = getResourceViewFormat(resViewDesc.format); - desc.width = resViewDesc.width; - desc.height = resViewDesc.height; - desc.depth = resViewDesc.depth; - desc.firstMipmapLevel = resViewDesc.firstMipmapLevel; - desc.lastMipmapLevel = resViewDesc.lastMipmapLevel; - desc.firstLayer = resViewDesc.firstLayer; - desc.lastLayer = resViewDesc.lastLayer; - - return desc; -} - -inline -size_t getElementSize(const hipChannelFormatDesc &desc) { - return (desc.x / 8) * getNumChannels(desc); -} -}; diff --git a/src/hipamd/src/hip_device.cpp b/src/hipamd/src/hip_device.cpp deleted file mode 100644 index 3488bc21bd..0000000000 --- a/src/hipamd/src/hip_device.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/* Copyright (c) 2018-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include - -#include "hip_internal.hpp" - -namespace hip { - -// ================================================================================================ -amd::HostQueue* Device::NullStream(bool skip_alloc) { - amd::HostQueue* null_queue = null_stream_.asHostQueue(skip_alloc); - if (null_queue == nullptr) { - return nullptr; - } - // Wait for all active streams before executing commands on the default - iHipWaitActiveStreams(null_queue); - return null_queue; -} - -} - -hipError_t hipDeviceGet(hipDevice_t *device, int deviceId) { - HIP_INIT_API(hipDeviceGet, device, deviceId); - - if (deviceId < 0 || - static_cast(deviceId) >= g_devices.size() || - device == nullptr) { - HIP_RETURN(hipErrorInvalidDevice); - } - *device = deviceId; - HIP_RETURN(hipSuccess); -}; - -hipError_t hipDeviceTotalMem (size_t *bytes, hipDevice_t device) { - - HIP_INIT_API(hipDeviceTotalMem, bytes, device); - - if (device < 0 || static_cast(device) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidDevice); - } - - if (bytes == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - auto* deviceHandle = g_devices[device]->devices()[0]; - const auto& info = deviceHandle->info(); - - *bytes = info.globalMemSize_; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceComputeCapability(int *major, int *minor, hipDevice_t device) { - - HIP_INIT_API(hipDeviceComputeCapability, major, minor, device); - - if (device < 0 || static_cast(device) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidDevice); - } - - if (major == nullptr || minor == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - auto* deviceHandle = g_devices[device]->devices()[0]; - const auto& isa = deviceHandle->isa(); - *major = isa.versionMajor(); - *minor = isa.versionMinor(); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceGetCount(int* count) { - HIP_INIT_API(hipDeviceGetCount, count); - - HIP_RETURN(ihipDeviceGetCount(count)); -} - -hipError_t ihipDeviceGetCount(int* count) { - if (count == nullptr) { - return hipErrorInvalidValue; - } - - // Get all available devices - *count = g_devices.size(); - - if (*count < 1) { - return hipErrorNoDevice; - } - - return hipSuccess; -} - -hipError_t hipDeviceGetName(char *name, int len, hipDevice_t device) { - - HIP_INIT_API(hipDeviceGetName, (void*)name, len, device); - - if (device < 0 || static_cast(device) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidDevice); - } - - if (name == nullptr || len <= 0) { - HIP_RETURN(hipErrorInvalidValue); - } - - auto* deviceHandle = g_devices[device]->devices()[0]; - const auto& info = deviceHandle->info(); - const auto nameLen = ::strlen(info.boardName_); - - // Make sure that the size of `dest` is big enough to hold `src` including - // trailing zero byte - if (nameLen > (cl_uint)(len - 1)) { - HIP_RETURN(hipErrorInvalidValue); - } - - ::strncpy(name, info.boardName_, (nameLen + 1)); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipGetDeviceProperties ( hipDeviceProp_t* props, hipDevice_t device ) { - HIP_INIT_API(hipGetDeviceProperties, props, device); - - if (props == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - if (unsigned(device) >= g_devices.size()) { - HIP_RETURN(hipErrorInvalidDevice); - } - auto* deviceHandle = g_devices[device]->devices()[0]; - - hipDeviceProp_t deviceProps = {0}; - - const auto& info = deviceHandle->info(); - const auto& isa = deviceHandle->isa(); - ::strncpy(deviceProps.name, info.boardName_, 128); - deviceProps.totalGlobalMem = info.globalMemSize_; - deviceProps.sharedMemPerBlock = info.localMemSizePerCU_; - deviceProps.regsPerBlock = info.availableRegistersPerCU_; - deviceProps.warpSize = info.wavefrontWidth_; - deviceProps.maxThreadsPerBlock = info.maxWorkGroupSize_; - deviceProps.maxThreadsDim[0] = info.maxWorkItemSizes_[0]; - deviceProps.maxThreadsDim[1] = info.maxWorkItemSizes_[1]; - deviceProps.maxThreadsDim[2] = info.maxWorkItemSizes_[2]; - deviceProps.maxGridSize[0] = INT32_MAX; - deviceProps.maxGridSize[1] = INT32_MAX; - deviceProps.maxGridSize[2] = INT32_MAX; - deviceProps.clockRate = info.maxEngineClockFrequency_ * 1000; - deviceProps.memoryClockRate = info.maxMemoryClockFrequency_ * 1000; - deviceProps.memoryBusWidth = info.globalMemChannels_; - deviceProps.totalConstMem = info.maxConstantBufferSize_; - deviceProps.major = isa.versionMajor(); - deviceProps.minor = isa.versionMinor(); - deviceProps.multiProcessorCount = info.maxComputeUnits_; - deviceProps.l2CacheSize = info.l2CacheSize_; - deviceProps.maxThreadsPerMultiProcessor = info.maxThreadsPerCU_; - deviceProps.computeMode = 0; - deviceProps.clockInstructionRate = info.timeStampFrequency_; - deviceProps.arch.hasGlobalInt32Atomics = 1; - deviceProps.arch.hasGlobalFloatAtomicExch = 1; - deviceProps.arch.hasSharedInt32Atomics = 1; - deviceProps.arch.hasSharedFloatAtomicExch = 1; - deviceProps.arch.hasFloatAtomicAdd = 1; - deviceProps.arch.hasGlobalInt64Atomics = 1; - deviceProps.arch.hasSharedInt64Atomics = 1; - deviceProps.arch.hasDoubles = 1; - deviceProps.arch.hasWarpVote = 1; - deviceProps.arch.hasWarpBallot = 1; - deviceProps.arch.hasWarpShuffle = 1; - deviceProps.arch.hasFunnelShift = 0; - deviceProps.arch.hasThreadFenceSystem = 1; - deviceProps.arch.hasSyncThreadsExt = 0; - deviceProps.arch.hasSurfaceFuncs = 0; - deviceProps.arch.has3dGrid = 1; - deviceProps.arch.hasDynamicParallelism = 0; - deviceProps.concurrentKernels = 1; - deviceProps.pciDomainID = info.pciDomainID; - deviceProps.pciBusID = info.deviceTopology_.pcie.bus; - deviceProps.pciDeviceID = info.deviceTopology_.pcie.device; - deviceProps.maxSharedMemoryPerMultiProcessor = info.localMemSizePerCU_; - deviceProps.canMapHostMemory = 1; - //FIXME: This should be removed, targets can have character names as well. - deviceProps.gcnArch = isa.versionMajor() * 100 + isa.versionMinor() * 10 + isa.versionStepping(); - sprintf(deviceProps.gcnArchName, "%s", isa.targetId()); - deviceProps.cooperativeLaunch = info.cooperativeGroups_; - deviceProps.cooperativeMultiDeviceLaunch = info.cooperativeMultiDeviceGroups_; - - deviceProps.cooperativeMultiDeviceUnmatchedFunc = info.cooperativeMultiDeviceGroups_; - deviceProps.cooperativeMultiDeviceUnmatchedGridDim = info.cooperativeMultiDeviceGroups_; - deviceProps.cooperativeMultiDeviceUnmatchedBlockDim = info.cooperativeMultiDeviceGroups_; - deviceProps.cooperativeMultiDeviceUnmatchedSharedMem = info.cooperativeMultiDeviceGroups_; - - deviceProps.maxTexture1DLinear = 16 * info.imageMaxBufferSize_; // Max pixel size is 16 bytes - deviceProps.maxTexture1D = info.image1DMaxWidth_; - deviceProps.maxTexture2D[0] = info.image2DMaxWidth_; - deviceProps.maxTexture2D[1] = info.image2DMaxHeight_; - deviceProps.maxTexture3D[0] = info.image3DMaxWidth_; - deviceProps.maxTexture3D[1] = info.image3DMaxHeight_; - deviceProps.maxTexture3D[2] = info.image3DMaxDepth_; - deviceProps.hdpMemFlushCntl = info.hdpMemFlushCntl; - deviceProps.hdpRegFlushCntl = info.hdpRegFlushCntl; - - deviceProps.memPitch = info.maxMemAllocSize_; - deviceProps.textureAlignment = info.imageBaseAddressAlignment_; - deviceProps.texturePitchAlignment = info.imagePitchAlignment_; - deviceProps.kernelExecTimeoutEnabled = 0; - deviceProps.ECCEnabled = info.errorCorrectionSupport_? 1:0; - deviceProps.isLargeBar = info.largeBar_ ? 1 : 0; - deviceProps.asicRevision = info.asicRevision_; - - // HMM capabilities - deviceProps.managedMemory = info.hmmSupported_; - deviceProps.concurrentManagedAccess = info.hmmSupported_; - deviceProps.directManagedMemAccessFromHost = info.hmmDirectHostAccess_; - deviceProps.pageableMemoryAccess = info.hmmCpuMemoryAccessible_; - deviceProps.pageableMemoryAccessUsesHostPageTables = info.hostUnifiedMemory_; - - *props = deviceProps; - HIP_RETURN(hipSuccess); -} diff --git a/src/hipamd/src/hip_device_runtime.cpp b/src/hipamd/src/hip_device_runtime.cpp deleted file mode 100755 index 560821c033..0000000000 --- a/src/hipamd/src/hip_device_runtime.cpp +++ /dev/null @@ -1,563 +0,0 @@ -/* Copyright (c) 2018-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include - -#include "hip_internal.hpp" - -hipError_t hipChooseDevice(int* device, const hipDeviceProp_t* properties) { - - HIP_INIT_API(hipChooseDevice, device, properties); - - if (device == nullptr || properties == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - *device = 0; - cl_uint maxMatchedCount = 0; - int count = 0; - ihipDeviceGetCount(&count); - - for (cl_int i = 0; i< count; ++i) { - hipDeviceProp_t currentProp = {0}; - cl_uint validPropCount = 0; - cl_uint matchedCount = 0; - hipError_t err = hipGetDeviceProperties(¤tProp, i); - if (properties->major != 0) { - validPropCount++; - if(currentProp.major >= properties->major) { - matchedCount++; - } - } - if (properties->minor != 0) { - validPropCount++; - if(currentProp.minor >= properties->minor) { - matchedCount++; - } - } - if(properties->totalGlobalMem != 0) { - validPropCount++; - if(currentProp.totalGlobalMem >= properties->totalGlobalMem) { - matchedCount++; - } - } - if(properties->sharedMemPerBlock != 0) { - validPropCount++; - if(currentProp.sharedMemPerBlock >= properties->sharedMemPerBlock) { - matchedCount++; - } - } - if(properties->maxThreadsPerBlock != 0) { - validPropCount++; - if(currentProp.maxThreadsPerBlock >= properties->maxThreadsPerBlock ) { - matchedCount++; - } - } - if(properties->totalConstMem != 0) { - validPropCount++; - if(currentProp.totalConstMem >= properties->totalConstMem ) { - matchedCount++; - } - } - if(properties->multiProcessorCount != 0) { - validPropCount++; - if(currentProp.multiProcessorCount >= - properties->multiProcessorCount ) { - matchedCount++; - } - } - if(properties->maxThreadsPerMultiProcessor != 0) { - validPropCount++; - if(currentProp.maxThreadsPerMultiProcessor >= - properties->maxThreadsPerMultiProcessor ) { - matchedCount++; - } - } - if(properties->memoryClockRate != 0) { - validPropCount++; - if(currentProp.memoryClockRate >= properties->memoryClockRate ) { - matchedCount++; - } - } - if(properties->memoryBusWidth != 0) { - validPropCount++; - if(currentProp.memoryBusWidth >= properties->memoryBusWidth ) { - matchedCount++; - } - } - if(properties->l2CacheSize != 0) { - validPropCount++; - if(currentProp.l2CacheSize >= properties->l2CacheSize ) { - matchedCount++; - } - } - if(properties->regsPerBlock != 0) { - validPropCount++; - if(currentProp.regsPerBlock >= properties->regsPerBlock ) { - matchedCount++; - } - } - if(properties->maxSharedMemoryPerMultiProcessor != 0) { - validPropCount++; - if(currentProp.maxSharedMemoryPerMultiProcessor >= - properties->maxSharedMemoryPerMultiProcessor ) { - matchedCount++; - } - } - if(properties->warpSize != 0) { - validPropCount++; - if(currentProp.warpSize >= properties->warpSize ) { - matchedCount++; - } - } - if(validPropCount == matchedCount) { - *device = matchedCount > maxMatchedCount ? i : *device; - maxMatchedCount = std::max(matchedCount, maxMatchedCount); - } - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceGetAttribute(int* pi, hipDeviceAttribute_t attr, int device) { - - HIP_INIT_API(hipDeviceGetAttribute, pi, attr, device); - - if (pi == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - int count = 0; - ihipDeviceGetCount(&count); - if (device < 0 || device >= count) { - HIP_RETURN(hipErrorInvalidDevice); - } - - //FIXME: should we cache the props, or just select from deviceHandle->info_? - hipDeviceProp_t prop = {0}; - hipError_t err = hipGetDeviceProperties(&prop, device); - if (err != hipSuccess) { - HIP_RETURN(err); - } - - switch (attr) { - case hipDeviceAttributeMaxThreadsPerBlock: - *pi = prop.maxThreadsPerBlock; - break; - case hipDeviceAttributeMaxBlockDimX: - *pi = prop.maxThreadsDim[0]; - break; - case hipDeviceAttributeMaxBlockDimY: - *pi = prop.maxThreadsDim[1]; - break; - case hipDeviceAttributeMaxBlockDimZ: - *pi = prop.maxThreadsDim[2]; - break; - case hipDeviceAttributeMaxGridDimX: - *pi = prop.maxGridSize[0]; - break; - case hipDeviceAttributeMaxGridDimY: - *pi = prop.maxGridSize[1]; - break; - case hipDeviceAttributeMaxGridDimZ: - *pi = prop.maxGridSize[2]; - break; - case hipDeviceAttributeMaxSharedMemoryPerBlock: - *pi = prop.sharedMemPerBlock; - break; - case hipDeviceAttributeTotalConstantMemory: - *pi = prop.totalConstMem; - break; - case hipDeviceAttributeWarpSize: - *pi = prop.warpSize; - break; - case hipDeviceAttributeMaxRegistersPerBlock: - *pi = prop.regsPerBlock; - break; - case hipDeviceAttributeClockRate: - *pi = prop.clockRate; - break; - case hipDeviceAttributeMemoryClockRate: - *pi = prop.memoryClockRate; - break; - case hipDeviceAttributeMemoryBusWidth: - *pi = prop.memoryBusWidth; - break; - case hipDeviceAttributeMultiprocessorCount: - *pi = prop.multiProcessorCount; - break; - case hipDeviceAttributeComputeMode: - *pi = prop.computeMode; - break; - case hipDeviceAttributeL2CacheSize: - *pi = prop.l2CacheSize; - break; - case hipDeviceAttributeMaxThreadsPerMultiProcessor: - *pi = prop.maxThreadsPerMultiProcessor; - break; - case hipDeviceAttributeComputeCapabilityMajor: - *pi = prop.major; - break; - case hipDeviceAttributeComputeCapabilityMinor: - *pi = prop.minor; - break; - case hipDeviceAttributePciBusId: - *pi = prop.pciBusID; - break; - case hipDeviceAttributeConcurrentKernels: - *pi = prop.concurrentKernels; - break; - case hipDeviceAttributePciDeviceId: - *pi = prop.pciDeviceID; - break; - case hipDeviceAttributeMaxSharedMemoryPerMultiprocessor: - *pi = prop.maxSharedMemoryPerMultiProcessor; - break; - case hipDeviceAttributeIsMultiGpuBoard: - *pi = prop.isMultiGpuBoard; - break; - case hipDeviceAttributeCooperativeLaunch: - *pi = prop.cooperativeLaunch; - break; - case hipDeviceAttributeCooperativeMultiDeviceLaunch: - *pi = prop.cooperativeMultiDeviceLaunch; - break; - case hipDeviceAttributeIntegrated: - *pi = prop.integrated; - break; - case hipDeviceAttributeMaxTexture1DWidth: - *pi = prop.maxTexture1D; - break; - case hipDeviceAttributeMaxTexture2DWidth: - *pi = prop.maxTexture2D[0]; - break; - case hipDeviceAttributeMaxTexture2DHeight: - *pi = prop.maxTexture2D[1]; - break; - case hipDeviceAttributeMaxTexture3DWidth: - *pi = prop.maxTexture3D[0]; - break; - case hipDeviceAttributeMaxTexture3DHeight: - *pi = prop.maxTexture3D[1]; - break; - case hipDeviceAttributeMaxTexture3DDepth: - *pi = prop.maxTexture3D[2]; - break; - case hipDeviceAttributeHdpMemFlushCntl: - *reinterpret_cast(pi) = prop.hdpMemFlushCntl; - break; - case hipDeviceAttributeHdpRegFlushCntl: - *reinterpret_cast(pi) = prop.hdpRegFlushCntl; - break; - case hipDeviceAttributeMaxPitch: - *pi = prop.memPitch; - break; - case hipDeviceAttributeTextureAlignment: - *pi = prop.textureAlignment; - break; - case hipDeviceAttributeTexturePitchAlignment: - *pi = prop.texturePitchAlignment; - break; - case hipDeviceAttributeKernelExecTimeout: - *pi = prop.kernelExecTimeoutEnabled; - break; - case hipDeviceAttributeCanMapHostMemory: - *pi = prop.canMapHostMemory; - break; - case hipDeviceAttributeEccEnabled: - *pi = prop.ECCEnabled; - break; - case hipDeviceAttributeCooperativeMultiDeviceUnmatchedFunc: - *pi = prop.cooperativeMultiDeviceUnmatchedFunc; - break; - case hipDeviceAttributeCooperativeMultiDeviceUnmatchedGridDim: - *pi = prop.cooperativeMultiDeviceUnmatchedGridDim; - break; - case hipDeviceAttributeCooperativeMultiDeviceUnmatchedBlockDim: - *pi = prop.cooperativeMultiDeviceUnmatchedBlockDim; - break; - case hipDeviceAttributeCooperativeMultiDeviceUnmatchedSharedMem: - *pi = prop.cooperativeMultiDeviceUnmatchedSharedMem; - break; - case hipDeviceAttributeAsicRevision: - *pi = prop.asicRevision; - break; - case hipDeviceAttributeManagedMemory: - *pi = prop.managedMemory; - break; - case hipDeviceAttributeDirectManagedMemAccessFromHost: - *pi = prop.directManagedMemAccessFromHost; - break; - case hipDeviceAttributeConcurrentManagedAccess: - *pi = prop.concurrentManagedAccess; - break; - case hipDeviceAttributePageableMemoryAccess: - *pi = prop.pageableMemoryAccess; - break; - case hipDeviceAttributePageableMemoryAccessUsesHostPageTables: - *pi = prop.pageableMemoryAccessUsesHostPageTables; - break; - case hipDeviceAttributeCanUseStreamWaitValue: - // hipStreamWaitValue64() and hipStreamWaitValue32() support - *pi = g_devices[device]->devices()[0]->info().aqlBarrierValue_; - break; - default: - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceGetByPCIBusId(int* device, const char*pciBusIdstr) { - - HIP_INIT_API(hipDeviceGetByPCIBusId, device, pciBusIdstr); - - if (device == nullptr || pciBusIdstr == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - int pciBusID = -1; - int pciDeviceID = -1; - int pciDomainID = -1; - bool found = false; - if (sscanf (pciBusIdstr, "%04x:%02x:%02x", &pciDomainID, &pciBusID, &pciDeviceID) == 0x3) { - int count = 0; - ihipDeviceGetCount(&count); - for (cl_int i = 0; i < count; i++) { - hipDevice_t dev; - hipDeviceGet(&dev, i); - hipDeviceProp_t prop; - hipGetDeviceProperties(&prop, dev); - - if ((pciBusID == prop.pciBusID) && (pciDomainID == prop.pciDomainID) - && (pciDeviceID == prop.pciDeviceID)) { - *device = i; - found = true; - break; - } - } - } - if (!found) { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceGetCacheConfig ( hipFuncCache_t * cacheConfig ) { - HIP_INIT_API(hipDeviceGetCacheConfig, cacheConfig); - - if(cacheConfig == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - *cacheConfig = hipFuncCache_t(); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceGetLimit ( size_t* pValue, hipLimit_t limit ) { - - HIP_INIT_API(hipDeviceGetLimit, pValue, limit); - - if(pValue == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - if(limit == hipLimitMallocHeapSize) { - hipDeviceProp_t prop; - hipGetDeviceProperties(&prop, ihipGetDevice()); - - *pValue = prop.totalGlobalMem; - HIP_RETURN(hipSuccess); - } else { - HIP_RETURN(hipErrorUnsupportedLimit); - } -} - -hipError_t hipDeviceGetPCIBusId ( char* pciBusId, int len, int device ) { - - HIP_INIT_API(hipDeviceGetPCIBusId, (void*)pciBusId, len, device); - - int count; - ihipDeviceGetCount(&count); - if (device < 0 || device >= count) { - HIP_RETURN(hipErrorInvalidDevice); - } - - if (pciBusId == nullptr || len <= 0) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipDeviceProp_t prop; - hipGetDeviceProperties(&prop, device); - - snprintf (pciBusId, len, "%04x:%02x:%02x.0", - prop.pciDomainID, - prop.pciBusID, - prop.pciDeviceID); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceGetSharedMemConfig ( hipSharedMemConfig * pConfig ) { - HIP_INIT_API(hipDeviceGetSharedMemConfig, pConfig); - if (pConfig == nullptr) { - return HIP_RETURN(hipErrorInvalidValue); - } - *pConfig = hipSharedMemBankSizeFourByte; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceReset ( void ) { - HIP_INIT_API(hipDeviceReset); - - /* FIXME */ - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceSetCacheConfig ( hipFuncCache_t cacheConfig ) { - HIP_INIT_API(hipDeviceSetCacheConfig, cacheConfig); - - // No way to set cache config yet. - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceSetLimit ( hipLimit_t limit, size_t value ) { - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipDeviceSetSharedMemConfig ( hipSharedMemConfig config ) { - HIP_INIT_API(hipDeviceSetSharedMemConfig, config); - - // No way to set cache config yet. - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceSynchronize ( void ) { - HIP_INIT_API(hipDeviceSynchronize); - - amd::HostQueue* queue = hip::getNullStream(); - - if (!queue) { - HIP_RETURN(hipErrorOutOfMemory); - } - - queue->finish(); - - hip::Stream::syncNonBlockingStreams(); - - HIP_RETURN(hipSuccess); -} - -int ihipGetDevice() { - hip::Device* device = hip::getCurrentDevice(); - if(device == nullptr){ - return -1; - } - return device->deviceId(); -} - -hipError_t hipGetDevice ( int* deviceId ) { - HIP_INIT_API(hipGetDevice, deviceId); - - if (deviceId != nullptr) { - int dev = ihipGetDevice(); - if (dev == -1) { - HIP_RETURN(hipErrorNoDevice); - } - *deviceId = dev; - HIP_RETURN(hipSuccess); - } else { - HIP_RETURN(hipErrorInvalidValue); - } -} - -hipError_t hipGetDeviceCount ( int* count ) { - HIP_INIT_API(hipGetDeviceCount, count); - - HIP_RETURN(ihipDeviceGetCount(count)); -} - -hipError_t hipGetDeviceFlags ( unsigned int* flags ) { - HIP_INIT_API(hipGetDeviceFlags, flags); - if (flags == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - *flags = hip::getCurrentDevice()->getFlags(); - HIP_RETURN(hipSuccess); -} - -hipError_t hipSetDevice ( int device ) { - HIP_INIT_API(hipSetDevice, device); - - if (static_cast(device) < g_devices.size()) { - hip::setCurrentDevice(device); - - HIP_RETURN(hipSuccess); - } - HIP_RETURN(hipErrorInvalidDevice); -} - -hipError_t hipSetDeviceFlags ( unsigned int flags ) { - HIP_INIT_API(hipSetDeviceFlags, flags); - - constexpr uint32_t supportedFlags = - hipDeviceScheduleMask | hipDeviceMapHost | hipDeviceLmemResizeToMax; - - if (flags & ~supportedFlags) { - HIP_RETURN(hipErrorInvalidValue); - } - - amd::Device* device = hip::getCurrentDevice()->devices()[0]; - switch (flags & hipDeviceScheduleMask) { - case hipDeviceScheduleAuto: - // Current behavior is different from the spec, due to MT usage in runtime - if (hip::host_device->devices().size() >= std::thread::hardware_concurrency()) { - device->SetActiveWait(false); - break; - } - // Fall through for active wait... - case hipDeviceScheduleSpin: - case hipDeviceScheduleYield: - // The both options falls into yield, because MT usage in runtime - device->SetActiveWait(true); - break; - case hipDeviceScheduleBlockingSync: - device->SetActiveWait(false); - break; - default: - break; - } - hip::getCurrentDevice()->setFlags(flags & hipDeviceScheduleMask); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipSetValidDevices ( int* device_arr, int len ) { - HIP_INIT_API(hipSetValidDevices, device_arr, len); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} diff --git a/src/hipamd/src/hip_error.cpp b/src/hipamd/src/hip_error.cpp deleted file mode 100644 index 7ecedff8d4..0000000000 --- a/src/hipamd/src/hip_error.cpp +++ /dev/null @@ -1,176 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include - -#include "hip_internal.hpp" - -hipError_t hipGetLastError() -{ - HIP_INIT_API(hipGetLastError); - hipError_t err = hip::g_lastError; - hip::g_lastError = hipSuccess; - return err; -} - -hipError_t hipPeekAtLastError() -{ - HIP_INIT_API(hipPeekAtLastError); - hipError_t err = hip::g_lastError; - HIP_RETURN(err); -} - -const char *hipGetErrorName(hipError_t hip_error) -{ - switch (hip_error) { - case hipSuccess: - return "hipSuccess"; - case hipErrorInvalidValue: - return "hipErrorInvalidValue"; - case hipErrorOutOfMemory: - return "hipErrorOutOfMemory"; - case hipErrorNotInitialized: - return "hipErrorNotInitialized"; - case hipErrorDeinitialized: - return "hipErrorDeinitialized"; - case hipErrorProfilerDisabled: - return "hipErrorProfilerDisabled"; - case hipErrorProfilerNotInitialized: - return "hipErrorProfilerNotInitialized"; - case hipErrorProfilerAlreadyStarted: - return "hipErrorProfilerAlreadyStarted"; - case hipErrorProfilerAlreadyStopped: - return "hipErrorProfilerAlreadyStopped"; - case hipErrorInvalidConfiguration: - return "hipErrorInvalidConfiguration"; - case hipErrorInvalidSymbol: - return "hipErrorInvalidSymbol"; - case hipErrorInvalidDevicePointer: - return "hipErrorInvalidDevicePointer"; - case hipErrorInvalidMemcpyDirection: - return "hipErrorInvalidMemcpyDirection"; - case hipErrorInsufficientDriver: - return "hipErrorInsufficientDriver"; - case hipErrorMissingConfiguration: - return "hipErrorMissingConfiguration"; - case hipErrorPriorLaunchFailure: - return "hipErrorPriorLaunchFailure"; - case hipErrorInvalidDeviceFunction: - return "hipErrorInvalidDeviceFunction"; - case hipErrorNoDevice: - return "hipErrorNoDevice"; - case hipErrorInvalidDevice: - return "hipErrorInvalidDevice"; - case hipErrorInvalidPitchValue: - return "hipErrorInvalidPitchValue"; - case hipErrorInvalidImage: - return "hipErrorInvalidImage"; - case hipErrorInvalidContext: - return "hipErrorInvalidContext"; - case hipErrorContextAlreadyCurrent: - return "hipErrorContextAlreadyCurrent"; - case hipErrorMapFailed: - return "hipErrorMapFailed"; - case hipErrorUnmapFailed: - return "hipErrorUnmapFailed"; - case hipErrorArrayIsMapped: - return "hipErrorArrayIsMapped"; - case hipErrorAlreadyMapped: - return "hipErrorAlreadyMapped"; - case hipErrorNoBinaryForGpu: - return "hipErrorNoBinaryForGpu"; - case hipErrorAlreadyAcquired: - return "hipErrorAlreadyAcquired"; - case hipErrorNotMapped: - return "hipErrorNotMapped"; - case hipErrorNotMappedAsArray: - return "hipErrorNotMappedAsArray"; - case hipErrorNotMappedAsPointer: - return "hipErrorNotMappedAsPointer"; - case hipErrorECCNotCorrectable: - return "hipErrorECCNotCorrectable"; - case hipErrorUnsupportedLimit: - return "hipErrorUnsupportedLimit"; - case hipErrorContextAlreadyInUse: - return "hipErrorContextAlreadyInUse"; - case hipErrorPeerAccessUnsupported: - return "hipErrorPeerAccessUnsupported"; - case hipErrorInvalidKernelFile: - return "hipErrorInvalidKernelFile"; - case hipErrorInvalidGraphicsContext: - return "hipErrorInvalidGraphicsContext"; - case hipErrorInvalidSource: - return "hipErrorInvalidSource"; - case hipErrorFileNotFound: - return "hipErrorFileNotFound"; - case hipErrorSharedObjectSymbolNotFound: - return "hipErrorSharedObjectSymbolNotFound"; - case hipErrorSharedObjectInitFailed: - return "hipErrorSharedObjectInitFailed"; - case hipErrorOperatingSystem: - return "hipErrorOperatingSystem"; - case hipErrorInvalidHandle: - return "hipErrorInvalidHandle"; - case hipErrorNotFound: - return "hipErrorNotFound"; - case hipErrorNotReady: - return "hipErrorNotReady"; - case hipErrorIllegalAddress: - return "hipErrorIllegalAddress"; - case hipErrorLaunchOutOfResources: - return "hipErrorLaunchOutOfResources"; - case hipErrorLaunchTimeOut: - return "hipErrorLaunchTimeOut"; - case hipErrorPeerAccessAlreadyEnabled: - return "hipErrorPeerAccessAlreadyEnabled"; - case hipErrorPeerAccessNotEnabled: - return "hipErrorPeerAccessNotEnabled"; - case hipErrorSetOnActiveProcess: - return "hipErrorSetOnActiveProcess"; - case hipErrorAssert: - return "hipErrorAssert"; - case hipErrorHostMemoryAlreadyRegistered: - return "hipErrorHostMemoryAlreadyRegistered"; - case hipErrorHostMemoryNotRegistered: - return "hipErrorHostMemoryNotRegistered"; - case hipErrorLaunchFailure: - return "hipErrorLaunchFailure"; - case hipErrorNotSupported: - return "hipErrorNotSupported"; - case hipErrorUnknown: - return "hipErrorUnknown"; - case hipErrorRuntimeMemory: - return "hipErrorRuntimeMemory"; - case hipErrorRuntimeOther: - return "hipErrorRuntimeOther"; - case hipErrorCooperativeLaunchTooLarge: - return "hipErrorCooperativeLaunchTooLarge"; - case hipErrorTbd: - return "hipErrorTbd"; - default: - return "hipErrorUnknown"; - }; -} - -const char *hipGetErrorString(hipError_t hip_error) -{ - return hipGetErrorName(hip_error); -} - diff --git a/src/hipamd/src/hip_event.cpp b/src/hipamd/src/hip_event.cpp deleted file mode 100755 index 94904d576d..0000000000 --- a/src/hipamd/src/hip_event.cpp +++ /dev/null @@ -1,431 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include - -#include "hip_event.hpp" - -void ipcEventCallback(hipStream_t stream, hipError_t status, void* user_data) -{ - std::atomic *signal = reinterpret_cast*>(user_data); - signal->store(0); - return; -} - -namespace hip { - -bool Event::ready() { - if (event_->status() != CL_COMPLETE) { - event_->notifyCmdQueue(); - } - - return (event_->status() == CL_COMPLETE); -} - -hipError_t Event::query() { - amd::ScopedLock lock(lock_); - - // If event is not recorded, event_ is null, hence return hipSuccess - if (event_ == nullptr) { - return hipSuccess; - } - - return ready() ? hipSuccess : hipErrorNotReady; -} - -hipError_t Event::synchronize() { - amd::ScopedLock lock(lock_); - - // If event is not recorded, event_ is null, hence return hipSuccess - if (event_ == nullptr) { - return hipSuccess; - } - - event_->awaitCompletion(); - - return hipSuccess; -} - -hipError_t Event::elapsedTime(Event& eStop, float& ms) { - amd::ScopedLock startLock(lock_); - - if (this == &eStop) { - if (event_ == nullptr) { - return hipErrorInvalidHandle; - } - - if (flags & hipEventDisableTiming) { - return hipErrorInvalidHandle; - } - - if (!ready()) { - return hipErrorNotReady; - } - - ms = 0.f; - return hipSuccess; - } - amd::ScopedLock stopLock(eStop.lock_); - - if (event_ == nullptr || - eStop.event_ == nullptr) { - return hipErrorInvalidHandle; - } - - if ((flags | eStop.flags) & hipEventDisableTiming) { - return hipErrorInvalidHandle; - } - - if (!ready() || !eStop.ready()) { - return hipErrorNotReady; - } - - if (event_ == eStop.event_ && recorded_ && eStop.recorded_) { - // Events are the same, which indicates the stream is empty and likely - // eventRecord is called on another stream. For such cases insert and measure a - // marker. - amd::Command* command = new amd::Marker(*event_->command().queue(), kMarkerDisableFlush); - command->enqueue(); - command->awaitCompletion(); - ms = static_cast(static_cast(command->event().profilingInfo().end_) - time())/1000000.f; - command->release(); - } else { - ms = static_cast(eStop.time() - time())/1000000.f; - } - return hipSuccess; -} - -int64_t Event::time() const { - assert(event_ != nullptr); - if (recorded_) { - return static_cast(event_->profilingInfo().end_); - } else { - return static_cast(event_->profilingInfo().start_); - } -} - -hipError_t Event::streamWait(amd::HostQueue* hostQueue, uint flags) { - if ((event_ == nullptr) || (event_->command().queue() == hostQueue)) { - return hipSuccess; - } - - amd::ScopedLock lock(lock_); - bool retain = false; - - if (!event_->notifyCmdQueue()) { - return hipErrorLaunchOutOfResources; - } - amd::Command::EventWaitList eventWaitList; - eventWaitList.push_back(event_); - - amd::Command* command = new amd::Marker(*hostQueue, kMarkerDisableFlush, eventWaitList); - if (command == NULL) { - return hipErrorOutOfMemory; - } - command->enqueue(); - command->release(); - - return hipSuccess; -} - -void Event::addMarker(amd::HostQueue* queue, amd::Command* command, bool record) { - // Keep the lock always at the beginning of this to avoid a race. SWDEV-277847 - amd::ScopedLock lock(lock_); - - if (command == nullptr) { - bool recordExplicitGpuTs = !queue->properties().test(CL_QUEUE_PROFILING_ENABLE) && - !(flags & hipEventDisableTiming); - // Always submit a EventMarker. This would submit a NOP with a signal. - command = new hip::EventMarker(*queue, !kMarkerDisableFlush, recordExplicitGpuTs); - command->enqueue(); - } - - if (event_ == &command->event()) return; - - if (event_ != nullptr) { - event_->release(); - } - - event_ = &command->event(); - recorded_ = record; -} - -} - -hipError_t ihipEventCreateWithFlags(hipEvent_t* event, unsigned flags) { - if (event == nullptr) { - return hipErrorInvalidValue; - } -#if !defined(_MSC_VER) - unsigned supportedFlags = hipEventDefault | hipEventBlockingSync | hipEventDisableTiming | - hipEventReleaseToDevice | hipEventReleaseToSystem | hipEventInterprocess; -#else - unsigned supportedFlags = hipEventDefault | hipEventBlockingSync | hipEventDisableTiming | - hipEventReleaseToDevice | hipEventReleaseToSystem; -#endif - const unsigned releaseFlags = (hipEventReleaseToDevice | hipEventReleaseToSystem); - - const bool illegalFlags = - (flags & ~supportedFlags) || // can't set any unsupported flags. - (flags & releaseFlags) == releaseFlags; // can't set both release flags - - if (!illegalFlags) { - hip::Event* e = new hip::Event(flags); - if (e == nullptr) { - return hipErrorOutOfMemory; - } - *event = reinterpret_cast(e); - } else { - return hipErrorInvalidValue; - } - return hipSuccess; -} - -hipError_t ihipEventQuery(hipEvent_t event) { - if (event == nullptr) { - return hipErrorInvalidHandle; - } - - hip::Event* e = reinterpret_cast(event); - if ((e->flags & hipEventInterprocess) && (e->ipc_evt_.ipc_shmem_)) { - int prev_read_idx = e->ipc_evt_.ipc_shmem_->read_index; - int offset = (prev_read_idx % IPC_SIGNALS_PER_EVENT); - if (e->ipc_evt_.ipc_shmem_->read_index < prev_read_idx+IPC_SIGNALS_PER_EVENT && e->ipc_evt_.ipc_shmem_->signal[offset] != 0) { - return hipErrorNotReady; - } - return hipSuccess; - } else { - return e->query(); - } -} - -hipError_t hipEventCreateWithFlags(hipEvent_t* event, unsigned flags) { - HIP_INIT_API(hipEventCreateWithFlags, event, flags); - HIP_RETURN(ihipEventCreateWithFlags(event, flags), *event); -} - -hipError_t hipEventCreate(hipEvent_t* event) { - HIP_INIT_API(hipEventCreate, event); - HIP_RETURN(ihipEventCreateWithFlags(event, 0), *event); -} - -hipError_t hipEventDestroy(hipEvent_t event) { - HIP_INIT_API(hipEventDestroy, event); - - if (event == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - - hip::Event* e = reinterpret_cast(event); - if ((e->flags & hipEventInterprocess) && (e->ipc_evt_.ipc_shmem_)) { - int owners = -- e->ipc_evt_.ipc_shmem_->owners; - // Make sure event is synchronized - hipEventSynchronize(event); - if (!amd::Os::MemoryUnmapFile(e->ipc_evt_.ipc_shmem_,sizeof(hip::ihipIpcEventShmem_t))) { - HIP_RETURN(hipErrorInvalidHandle); - } - } - delete e; - HIP_RETURN(hipSuccess); -} - -hipError_t hipEventElapsedTime(float *ms, hipEvent_t start, hipEvent_t stop) { - HIP_INIT_API(hipEventElapsedTime, ms, start, stop); - - if (start == nullptr || stop == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - - if (ms == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - hip::Event* eStart = reinterpret_cast(start); - hip::Event* eStop = reinterpret_cast(stop); - - if (eStart->deviceId() != eStop->deviceId()) { - HIP_RETURN(hipErrorInvalidHandle); - } - - HIP_RETURN(eStart->elapsedTime(*eStop, *ms), "Elapsed Time = ", *ms); -} - -// ================================================================================================ -bool createIpcEventShmemIfNeeded(hip::Event::ihipIpcEvent_t& ipc_evt) { -#if !defined(_MSC_VER) - if (ipc_evt.ipc_shmem_) { - // ipc_shmem_ already created, no need to create it again - return true; - } - char name_template[] = "/tmp/eventXXXXXX"; - int temp_fd = mkstemp(name_template); - - ipc_evt.ipc_name_ = name_template; - ipc_evt.ipc_name_.replace(0, 5, "/hip_"); - if (!amd::Os::MemoryMapFileTruncated(ipc_evt.ipc_name_.c_str(), - const_cast (reinterpret_cast(&(ipc_evt.ipc_shmem_))), sizeof(hip::ihipIpcEventShmem_t))) { - return false; - } - ipc_evt.ipc_shmem_->owners = 1; - ipc_evt.ipc_shmem_->read_index = -1; - ipc_evt.ipc_shmem_->write_index = 0; - for (uint32_t sig_idx = 0; sig_idx < IPC_SIGNALS_PER_EVENT; ++sig_idx) { - ipc_evt.ipc_shmem_->signal[sig_idx] = 0; - } - - close(temp_fd); - return true; -#else - return false; -#endif -} - -hipError_t hipEventRecord(hipEvent_t event, hipStream_t stream) { - HIP_INIT_API(hipEventRecord, event, stream); - - STREAM_CAPTURE(hipEventRecord, stream, event); - - if (event == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - - hip::Event* e = reinterpret_cast(event); - - amd::HostQueue* queue = hip::getQueue(stream); - - if (g_devices[e->deviceId()]->devices()[0] != &queue->device()) { - HIP_RETURN(hipErrorInvalidHandle); - } - - bool isRecorded = e->isRecorded(); - if ((e->flags & hipEventInterprocess) && !isRecorded) { - amd::Command* command = new amd::Marker(*queue, kMarkerDisableFlush); - amd::Event& tEvent = command->event(); - createIpcEventShmemIfNeeded(e->ipc_evt_); - int write_index = e->ipc_evt_.ipc_shmem_->write_index++; - int offset = write_index % IPC_SIGNALS_PER_EVENT; - while (e->ipc_evt_.ipc_shmem_->signal[offset] != 0) { - amd::Os::sleep(1); - } - // Lock signal. - e->ipc_evt_.ipc_shmem_->signal[offset] = 1; - e->ipc_evt_.ipc_shmem_->owners_device_id = e->deviceId(); - - std::atomic *signal = &e->ipc_evt_.ipc_shmem_->signal[offset]; - StreamCallback* cbo = new StreamCallback(stream, - reinterpret_cast (ipcEventCallback), signal, command); - if (!tEvent.setCallback(CL_COMPLETE, ihipStreamCallback,cbo)) { - command->release(); - return hipErrorInvalidHandle; - } - command->enqueue(); - tEvent.notifyCmdQueue(); - // Update read index to indicate new signal. - int expected = write_index - 1; - while (!e->ipc_evt_.ipc_shmem_->read_index.compare_exchange_weak(expected, write_index)) { - amd::Os::sleep(1); - } - } else { - e->addMarker(queue, nullptr, true); - } - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipEventSynchronize(hipEvent_t event) { - HIP_INIT_API(hipEventSynchronize, event); - - if (event == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - - hip::Event* e = reinterpret_cast(event); - if ((e->flags & hipEventInterprocess) && (e->ipc_evt_.ipc_shmem_)) { - int prev_read_idx = e->ipc_evt_.ipc_shmem_->read_index; - if (prev_read_idx >= 0) { - int offset = (prev_read_idx % IPC_SIGNALS_PER_EVENT); - while ((e->ipc_evt_.ipc_shmem_->read_index < prev_read_idx + IPC_SIGNALS_PER_EVENT) - && (e->ipc_evt_.ipc_shmem_->signal[offset] != 0)) { - amd::Os::sleep(1); - } - } - HIP_RETURN(hipSuccess); - } else { - HIP_RETURN(e->synchronize()); - } -} - -hipError_t hipEventQuery(hipEvent_t event) { - HIP_INIT_API(hipEventQuery, event); - HIP_RETURN(ihipEventQuery(event)); -} - -hipError_t hipIpcGetEventHandle(hipIpcEventHandle_t* handle, hipEvent_t event) { - HIP_INIT_API(hipIpcGetEventHandle, handle, event); -#if !defined(_MSC_VER) - if (handle == nullptr || event == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - hip::Event* e = reinterpret_cast(event); - if (!(e->flags & hipEventInterprocess)) { - HIP_RETURN(hipErrorInvalidConfiguration); - } - if (!createIpcEventShmemIfNeeded(e->ipc_evt_)) { - HIP_RETURN(hipErrorInvalidConfiguration); - } - ihipIpcEventHandle_t* iHandle = reinterpret_cast(handle); - memset(iHandle->shmem_name, 0, HIP_IPC_HANDLE_SIZE); - e->ipc_evt_.ipc_name_.copy(iHandle->shmem_name, std::string::npos); - HIP_RETURN(hipSuccess); -#else - assert(0 && "Unimplemented"); - HIP_RETURN(hipErrorNotSupported); -#endif -} - -hipError_t hipIpcOpenEventHandle(hipEvent_t* event, hipIpcEventHandle_t handle) { - HIP_INIT_API(NONE, event, handle); -#if !defined(_MSC_VER) - hipError_t hip_err = hipSuccess; - if (event == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - hip_err = ihipEventCreateWithFlags(event, hipEventDisableTiming | hipEventInterprocess); - if (hip_err != hipSuccess) { - HIP_RETURN(hip_err); - } - hip::Event* e = reinterpret_cast(*event); - ihipIpcEventHandle_t* iHandle = reinterpret_cast(&handle); - hip::Event::ihipIpcEvent_t& ipc_evt = e->ipc_evt_; - ipc_evt.ipc_name_ = iHandle->shmem_name; - if (!amd::Os::MemoryMapFileTruncated(ipc_evt.ipc_name_.c_str(), - (const void**) &(ipc_evt.ipc_shmem_), sizeof(hip::ihipIpcEventShmem_t))) { - HIP_RETURN(hipErrorInvalidValue); - } - - ipc_evt.ipc_shmem_->owners += 1; - e->setDeviceId(ipc_evt.ipc_shmem_->owners_device_id.load()); - - HIP_RETURN(hipSuccess); -#else - assert(0 && "Unimplemented"); - HIP_RETURN(hipErrorNotSupported); -#endif -} diff --git a/src/hipamd/src/hip_event.hpp b/src/hipamd/src/hip_event.hpp deleted file mode 100644 index c7d0ae5dc3..0000000000 --- a/src/hipamd/src/hip_event.hpp +++ /dev/null @@ -1,151 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef HIP_EVENT_H -#define HIP_EVENT_H - -#include "hip_internal.hpp" -#include "thread/monitor.hpp" - -// Internal structure for stream callback handler -class StreamCallback { - public: - StreamCallback(hipStream_t stream, hipStreamCallback_t callback, void* userData, - amd::Command* command) - : stream_(stream), callBack_(callback), - userData_(userData), command_(command) { - }; - hipStream_t stream_; - hipStreamCallback_t callBack_; - void* userData_; - amd::Command* command_; -}; - -void CL_CALLBACK ihipStreamCallback(cl_event event, cl_int command_exec_status, void* user_data); - - -namespace hip { - -class EventMarker: public amd::Marker { -public: - EventMarker(amd::HostQueue& queue, bool disableFlush, bool markerTs = false) - : amd::Marker(queue, disableFlush) { - profilingInfo_.enabled_ = true; - profilingInfo_.callback_ = nullptr; - profilingInfo_.marker_ts_ = markerTs; - profilingInfo_.clear(); - } -}; - -#define IPC_SIGNALS_PER_EVENT 32 -typedef struct ihipIpcEventShmem_s { - std::atomic owners; - std::atomic owners_device_id; - std::atomic read_index; - std::atomic write_index; - std::atomic signal[IPC_SIGNALS_PER_EVENT]; -} ihipIpcEventShmem_t; - -class Event { - /// event recorded on stream where capture is active - bool onCapture_; - /// capture stream where event is recorded - hipStream_t captureStream_; - /// Previous captured nodes before event record - std::vector nodesPrevToRecorded_; - - public: - Event(unsigned int flags) : flags(flags), lock_("hipEvent_t", true), - event_(nullptr), recorded_(false) { - // No need to init event_ here as addMarker does that - onCapture_ = false; - device_id_ = hip::getCurrentDevice()->deviceId(); // Created in current device ctx - } - - ~Event() { - if (event_ != nullptr) { - event_->release(); - } - } - unsigned int flags; - - hipError_t query(); - hipError_t synchronize(); - hipError_t elapsedTime(Event& stop, float& ms); - hipError_t streamWait(amd::HostQueue* queue, uint flags); - - void addMarker(amd::HostQueue* queue, amd::Command* command, bool record); - bool isRecorded() { return recorded_; } - amd::Monitor& lock() { return lock_; } - const int deviceId() { return device_id_; } - void setDeviceId(int id) { device_id_ = id; } - - /// End capture on this event - void EndCapture() { - onCapture_ = false; - captureStream_ = nullptr; - } - /// Start capture when waited on this event - void StartCapture(hipStream_t stream) { - onCapture_ = true; - captureStream_ = stream; - } - /// Get capture status of the graph - bool GetCaptureStatus() { return onCapture_; } - /// Get capture stream where event is recorded - hipStream_t GetCaptureStream() { return captureStream_; } - /// Set capture stream where event is recorded - void SetCaptureStream(hipStream_t stream) { captureStream_ = stream; } - /// Returns previous captured nodes before event record - std::vector GetNodesPrevToRecorded() const { return nodesPrevToRecorded_; } - /// Set last captured graph node before event record - void SetNodesPrevToRecorded(std::vector& graphNode) { - nodesPrevToRecorded_ = graphNode; - } - - // IPC Events - struct ihipIpcEvent_t { - std::string ipc_name_; - int ipc_fd_; - ihipIpcEventShmem_t* ipc_shmem_; - ihipIpcEvent_t(): ipc_name_("dummy"), ipc_fd_(0), ipc_shmem_(nullptr) { - } - void setipcname(const char* name) { - ipc_name_ = std::string(name); - } - }; - ihipIpcEvent_t ipc_evt_; -private: - amd::Monitor lock_; - amd::HostQueue* stream_; - amd::Event* event_; - int device_id_; - //! Flag to indicate hipEventRecord has been called. This is needed except for - //! hip*ModuleLaunchKernel API which takes start and stop events so no - //! hipEventRecord is called. Cleanup needed once those APIs are deprecated. - bool recorded_; - - bool ready(); - int64_t time() const; -}; - -}; - -#endif // HIP_EVEMT_H diff --git a/src/hipamd/src/hip_fatbin.cpp b/src/hipamd/src/hip_fatbin.cpp deleted file mode 100755 index 04ae382bdf..0000000000 --- a/src/hipamd/src/hip_fatbin.cpp +++ /dev/null @@ -1,158 +0,0 @@ -#include "hip_fatbin.hpp" - -#include "hip_code_object.hpp" - -namespace hip { - -FatBinaryDeviceInfo::~FatBinaryDeviceInfo() { - if (program_ != nullptr) { - program_->release(); - program_ = nullptr; - } -} - -FatBinaryInfo::FatBinaryInfo(const char* fname, const void* image) - : fdesc_(amd::Os::FDescInit()), fsize_(0), image_(image), uri_(std::string()) { - - if (fname != nullptr) { - fname_ = std::string(fname); - } else { - fname_ = std::string(); - } - - fatbin_dev_info_.resize(g_devices.size()); -} - -FatBinaryInfo::~FatBinaryInfo() { - - for (auto& fbd: fatbin_dev_info_) { - delete fbd; - } - - if (fdesc_ > 0) { - if (fsize_ && !amd::Os::MemoryUnmapFile(image_, fsize_)) { - guarantee(false, "Cannot unmap file"); - } - if (!amd::Os::CloseFileHandle(fdesc_)) { - guarantee(false, "Cannot close file"); - } - } - - fname_ = std::string(); - fdesc_ = amd::Os::FDescInit(); - fsize_ = 0; - image_ = nullptr; - uri_ = std::string(); -} - -hipError_t FatBinaryInfo::ExtractFatBinary(const std::vector& devices) { - hipError_t hip_error = hipSuccess; - std::vector> code_objs; - - // Copy device names for Extract Code object File - std::vector device_names; - device_names.reserve(devices.size()); - for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) { - device_names.push_back(devices[dev_idx]->devices()[0]->isa().isaName()); - } - - // We are given file name, get the file desc and file size - if (fname_.size() > 0) { - // Get File Handle & size of the file. - if (!amd::Os::GetFileHandle(fname_.c_str(), &fdesc_, &fsize_)) { - return hipErrorFileNotFound; - } - if (fsize_ == 0) { - return hipErrorInvalidKernelFile; - } - - // Extract the code object from file - hip_error = CodeObject::ExtractCodeObjectFromFile(fdesc_, fsize_, &image_, - device_names, code_objs); - - } else if (image_ != nullptr) { - // We are directly given image pointer directly, try to extract file desc & file Size - hip_error = CodeObject::ExtractCodeObjectFromMemory(image_, - device_names, code_objs, uri_); - } else { - return hipErrorInvalidValue; - } - - if (hip_error == hipErrorNoBinaryForGpu) { - guarantee(false, "hipErrorNoBinaryForGpu: Couldn't find binary for current devices!"); - return hip_error; - } - - if (hip_error == hipErrorInvalidKernelFile) { - for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) { - // the image type is no CLANG_OFFLOAD_BUNDLER, image for current device directly passed - fatbin_dev_info_[devices[dev_idx]->deviceId()] - = new FatBinaryDeviceInfo(image_, CodeObject::ElfSize(image_), 0); - } - } else if(hip_error == hipSuccess) { - for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) { - // Calculate the offset wrt binary_image and the original image - size_t offset_l - = (reinterpret_cast
(const_cast(code_objs[dev_idx].first)) - - reinterpret_cast
(const_cast(image_))); - - fatbin_dev_info_[devices[dev_idx]->deviceId()] - = new FatBinaryDeviceInfo(code_objs[dev_idx].first, code_objs[dev_idx].second, offset_l); - } - } - - for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) { - fatbin_dev_info_[devices[dev_idx]->deviceId()]->program_ - = new amd::Program(*devices[dev_idx]->asContext()); - if (fatbin_dev_info_[devices[dev_idx]->deviceId()]->program_ == NULL) { - return hipErrorOutOfMemory; - } - } - - return hipSuccess; -} - -hipError_t FatBinaryInfo::AddDevProgram(const int device_id) { - // Device Id bounds Check - DeviceIdCheck(device_id); - - FatBinaryDeviceInfo* fbd_info = fatbin_dev_info_[device_id]; - // If fat binary was already added, skip this step and return success - if (fbd_info->add_dev_prog_ == false) { - amd::Context* ctx = g_devices[device_id]->asContext(); - if (CL_SUCCESS != fbd_info->program_->addDeviceProgram(*ctx->devices()[0], - fbd_info->binary_image_, - fbd_info->binary_size_, false, - nullptr, nullptr, fdesc_, - fbd_info->binary_offset_, uri_)) { - return hipErrorInvalidKernelFile; - } - fbd_info->add_dev_prog_ = true; - } - return hipSuccess; -} - -hipError_t FatBinaryInfo::BuildProgram(const int device_id) { - - // Device Id Check and Add DeviceProgram if not added so far - DeviceIdCheck(device_id); - IHIP_RETURN_ONFAIL(AddDevProgram(device_id)); - - // If Program was already built skip this step and return success - FatBinaryDeviceInfo* fbd_info = fatbin_dev_info_[device_id]; - if (fbd_info->prog_built_ == false) { - if(CL_SUCCESS != fbd_info->program_->build(g_devices[device_id]->devices(), - nullptr, nullptr, nullptr, - kOptionChangeable, kNewDevProg)) { - return hipErrorSharedObjectInitFailed; - } - fbd_info->prog_built_ = true; - } - - if (!fbd_info->program_->load()) { - return hipErrorSharedObjectInitFailed; - } - return hipSuccess; -} - -} //namespace : hip diff --git a/src/hipamd/src/hip_fatbin.hpp b/src/hipamd/src/hip_fatbin.hpp deleted file mode 100755 index 219a96e802..0000000000 --- a/src/hipamd/src/hip_fatbin.hpp +++ /dev/null @@ -1,87 +0,0 @@ -#ifndef HIP_FAT_BINARY_HPP -#define HIP_FAT_BINARY_HPP - -#include "hip/hip_runtime.h" -#include "hip/hip_runtime_api.h" -#include "hip_internal.hpp" -#include "platform/program.hpp" - -namespace hip { - -//Fat Binary Per Device info -class FatBinaryDeviceInfo { -public: - FatBinaryDeviceInfo (const void* binary_image, size_t binary_size, size_t binary_offset) - : binary_image_(binary_image), binary_size_(binary_size), - binary_offset_(binary_offset), program_(nullptr), - add_dev_prog_(false), prog_built_(false) {} - - ~FatBinaryDeviceInfo(); - -private: - const void* binary_image_; // binary image ptr - size_t binary_size_; // binary image size - size_t binary_offset_; // image offset from original - - amd::Program* program_; // reinterpreted as hipModule_t - friend class FatBinaryInfo; - - //Control Variables - bool add_dev_prog_; - bool prog_built_; -}; - - -// Fat Binary Info -class FatBinaryInfo { -public: - FatBinaryInfo(const char* fname, const void* image); - ~FatBinaryInfo(); - - // Loads Fat binary from file or image, unbundles COs for devices. - hipError_t ExtractFatBinary(const std::vector& devices); - hipError_t AddDevProgram(const int device_id); - hipError_t BuildProgram(const int device_id); - - - // Device Id bounds check - inline void DeviceIdCheck(const int device_id) const { - guarantee(device_id >= 0, "Invalid DeviceId less than 0"); - guarantee(static_cast(device_id) < fatbin_dev_info_.size(), "Invalid DeviceId, greater than no of fatbin device info!"); - } - - // Getter Methods - amd::Program* GetProgram(int device_id) { - DeviceIdCheck(device_id); - return fatbin_dev_info_[device_id]->program_; - } - - hipModule_t Module(int device_id) const { - DeviceIdCheck(device_id); - return reinterpret_cast(as_cl(fatbin_dev_info_[device_id]->program_)); - } - - hipError_t GetModule(int device_id, hipModule_t* hmod) const { - DeviceIdCheck(device_id); - *hmod = reinterpret_cast(as_cl(fatbin_dev_info_[device_id]->program_)); - return hipSuccess; - } - -private: - std::string fname_; // File name - amd::Os::FileDesc fdesc_; // File descriptor - size_t fsize_; // Total file size - - // Even when file is passed image will be mmapped till ~desctructor. - const void* image_; // Image - - // Only used for FBs where image is directly passed - std::string uri_; // Uniform resource indicator - - // Per Device Info, like corresponding binary ptr, size. - std::vector fatbin_dev_info_; -}; - -}; /* namespace hip */ - -#endif /* HIP_FAT_BINARY_HPP */ diff --git a/src/hipamd/src/hip_formatting.hpp b/src/hipamd/src/hip_formatting.hpp deleted file mode 100644 index 83f1467a7b..0000000000 --- a/src/hipamd/src/hip_formatting.hpp +++ /dev/null @@ -1,853 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ -#include -#include - -inline std::ostream& operator<<(std::ostream& os, const hipTextureFilterMode& s) { - switch (s) { - case hipFilterModePoint: - os << "hipFilterModePoint"; - break; - case hipFilterModeLinear: - os << "hipFilterModeLinear"; - break; - default: - os << "hipFilterModePoint"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipTextureReadMode& s) { - switch (s) { - case hipReadModeElementType: - os << "hipReadModeElementType"; - break; - case hipReadModeNormalizedFloat: - os << "hipReadModeNormalizedFloat"; - break; - default: - os << "hipReadModeElementType"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipTextureAddressMode& s) { - switch (s) { - case hipAddressModeWrap: - os << "hipAddressModeWrap"; - break; - case hipAddressModeClamp: - os << "hipAddressModeClamp"; - break; - case hipAddressModeMirror: - os << "hipAddressModeMirror"; - break; - case hipAddressModeBorder: - os << "hipAddressModeBorder"; - break; - default: - os << "hipAddressModeWrap"; - }; - return os; -} - - -inline std::ostream& operator<<(std::ostream& os, const hipMemcpyKind& s) { - switch (s) { - case hipMemcpyHostToHost: - os << "hipMemcpyHostToHost"; - break; - case hipMemcpyHostToDevice: - os << "hipMemcpyHostToDevice"; - break; - case hipMemcpyDeviceToHost: - os << "hipMemcpyDeviceToHost"; - break; - case hipMemcpyDeviceToDevice: - os << "hipMemcpyDeviceToDevice"; - break; - case hipMemcpyDefault: - os << "hipMemcpyDefault"; - break; - default: - os << "hipMemcpyDefault"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatKind& s) { - switch (s) { - case hipChannelFormatKindSigned: - os << "hipChannelFormatKindSigned"; - break; - case hipChannelFormatKindUnsigned: - os << "hipMemcpyHostToDevice"; - break; - case hipChannelFormatKindFloat: - os << "hipChannelFormatKindFloat"; - break; - case hipChannelFormatKindNone: - os << "hipChannelFormatKindNone"; - break; - default: - os << "hipChannelFormatKindNone"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipArray_Format& s) { - switch (s) { - case HIP_AD_FORMAT_UNSIGNED_INT8: - os << "HIP_AD_FORMAT_UNSIGNED_INT8"; - break; - case HIP_AD_FORMAT_UNSIGNED_INT16: - os << "HIP_AD_FORMAT_UNSIGNED_INT16"; - break; - case HIP_AD_FORMAT_UNSIGNED_INT32: - os << "HIP_AD_FORMAT_UNSIGNED_INT32"; - break; - case HIP_AD_FORMAT_SIGNED_INT8: - os << "HIP_AD_FORMAT_SIGNED_INT8"; - break; - case HIP_AD_FORMAT_SIGNED_INT16: - os << "HIP_AD_FORMAT_SIGNED_INT16"; - break; - case HIP_AD_FORMAT_SIGNED_INT32: - os << "HIP_AD_FORMAT_SIGNED_INT32"; - break; - case HIP_AD_FORMAT_HALF: - os << "HIP_AD_FORMAT_HALF"; - break; - case HIP_AD_FORMAT_FLOAT: - os << "HIP_AD_FORMAT_FLOAT"; - break; - default: - os << "HIP_AD_FORMAT_FLOAT"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipResourceViewFormat& s) { - switch (s) { - case hipResViewFormatNone: - os << "hipResViewFormatNone"; - break; - case hipResViewFormatUnsignedChar1: - os << "hipResViewFormatUnsignedChar1"; - break; - case hipResViewFormatUnsignedChar2: - os << "hipResViewFormatUnsignedChar2"; - break; - case hipResViewFormatUnsignedChar4: - os << "hipResViewFormatUnsignedChar4"; - break; - case hipResViewFormatSignedChar1: - os << "hipResViewFormatSignedChar1"; - break; - case hipResViewFormatSignedChar2: - os << "hipResViewFormatSignedChar2"; - break; - case hipResViewFormatSignedChar4: - os << "hipResViewFormatSignedChar4"; - break; - case hipResViewFormatUnsignedShort1: - os << "hipResViewFormatUnsignedShort1"; - break; - case hipResViewFormatUnsignedShort2: - os << "hipResViewFormatUnsignedShort2"; - break; - case hipResViewFormatUnsignedShort4: - os << "hipResViewFormatUnsignedShort4"; - break; - case hipResViewFormatSignedShort1: - os << "hipResViewFormatSignedShort1"; - break; - case hipResViewFormatSignedShort2: - os << "hipResViewFormatSignedShort2"; - break; - case hipResViewFormatSignedShort4: - os << "hipResViewFormatSignedShort4"; - break; - case hipResViewFormatUnsignedInt1: - os << "hipResViewFormatUnsignedInt1"; - break; - case hipResViewFormatUnsignedInt2: - os << "hipResViewFormatUnsignedInt2"; - break; - case hipResViewFormatUnsignedInt4: - os << "hipResViewFormatUnsignedInt4"; - break; - case hipResViewFormatSignedInt1: - os << "hipResViewFormatSignedInt1"; - break; - case hipResViewFormatSignedInt2: - os << "hipResViewFormatSignedInt2"; - break; - case hipResViewFormatSignedInt4: - os << "hipResViewFormatSignedInt4"; - break; - case hipResViewFormatHalf1: - os << "hipResViewFormatHalf1"; - break; - case hipResViewFormatHalf2: - os << "hipResViewFormatHalf2"; - break; - case hipResViewFormatHalf4: - os << "hipResViewFormatHalf4"; - break; - case hipResViewFormatFloat1: - os << "hipResViewFormatFloat1"; - break; - case hipResViewFormatFloat2: - os << "hipResViewFormatFloat2"; - break; - case hipResViewFormatFloat4: - os << "hipResViewFormatFloat4"; - break; - case hipResViewFormatUnsignedBlockCompressed1: - os << "hipResViewFormatUnsignedBlockCompressed1"; - break; - case hipResViewFormatUnsignedBlockCompressed2: - os << "hipResViewFormatUnsignedBlockCompressed2"; - break; - case hipResViewFormatUnsignedBlockCompressed3: - os << "hipResViewFormatUnsignedBlockCompressed3"; - break; - case hipResViewFormatUnsignedBlockCompressed4: - os << "hipResViewFormatUnsignedBlockCompressed4"; - break; - case hipResViewFormatSignedBlockCompressed4: - os << "hipResViewFormatSignedBlockCompressed4"; - break; - case hipResViewFormatUnsignedBlockCompressed5: - os << "hipResViewFormatUnsignedBlockCompressed5"; - break; - case hipResViewFormatSignedBlockCompressed5: - os << "hipResViewFormatSignedBlockCompressed5"; - break; - case hipResViewFormatUnsignedBlockCompressed6H: - os << "hipResViewFormatUnsignedBlockCompressed6H"; - break; - case hipResViewFormatSignedBlockCompressed6H: - os << "hipResViewFormatSignedBlockCompressed6H"; - break; - case hipResViewFormatUnsignedBlockCompressed7: - os << "hipResViewFormatUnsignedBlockCompressed7"; - break; - default: - os << "hipResViewFormatNone"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipFunction_attribute& s) { - switch (s) { - case HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK: - os << "HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK"; - break; - case HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES: - os << "HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES"; - break; - case HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES: - os << "HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES"; - break; - case HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES: - os << "HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES"; - break; - case HIP_FUNC_ATTRIBUTE_NUM_REGS: - os << "HIP_FUNC_ATTRIBUTE_NUM_REGS"; - break; - case HIP_FUNC_ATTRIBUTE_PTX_VERSION: - os << "HIP_FUNC_ATTRIBUTE_PTX_VERSION"; - break; - case HIP_FUNC_ATTRIBUTE_BINARY_VERSION: - os << "HIP_FUNC_ATTRIBUTE_BINARY_VERSION"; - break; - case HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA: - os << "HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA"; - break; - case HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES: - os << "HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES"; - break; - case HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT: - os << "HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT"; - break; - case HIP_FUNC_ATTRIBUTE_MAX: - os << "HIP_FUNC_ATTRIBUTE_MAX"; - break; - default: - os << "HIP_FUNC_ATTRIBUTE_MAX"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hiprtcResult& s) { - switch (s) { - case HIPRTC_SUCCESS: - os << "HIPRTC_SUCCESS"; - break; - case HIPRTC_ERROR_OUT_OF_MEMORY: - os << "HIPRTC_ERROR_OUT_OF_MEMORY"; - break; - case HIPRTC_ERROR_PROGRAM_CREATION_FAILURE: - os << "HIPRTC_ERROR_PROGRAM_CREATION_FAILURE"; - break; - case HIPRTC_ERROR_INVALID_INPUT: - os << "HIPRTC_ERROR_INVALID_INPUT"; - break; - case HIPRTC_ERROR_INVALID_PROGRAM: - os << "HIPRTC_ERROR_INVALID_PROGRAM"; - break; - case HIPRTC_ERROR_INVALID_OPTION: - os << "HIPRTC_ERROR_INVALID_OPTION"; - break; - case HIPRTC_ERROR_COMPILATION: - os << "HIPRTC_ERROR_COMPILATION"; - break; - case HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE: - os << "HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE"; - break; - case HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION: - os << "HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION"; - break; - case HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION: - os << "IPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION"; - break; - case HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID: - os << "HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID"; - break; - case HIPRTC_ERROR_INTERNAL_ERROR: - os << "HIPRTC_ERROR_INTERNAL_ERROR"; - break; - default: - os << "HIPRTC_ERROR_INTERNAL_ERROR"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipJitOption& s) { - switch (s) { - case hipJitOptionMaxRegisters: - os << "hipJitOptionMaxRegisters"; - break; - case hipJitOptionThreadsPerBlock: - os << "hipJitOptionThreadsPerBlock"; - break; - case hipJitOptionWallTime: - os << "hipJitOptionWallTime"; - break; - case hipJitOptionInfoLogBuffer: - os << "hipJitOptionInfoLogBuffer"; - break; - case hipJitOptionInfoLogBufferSizeBytes: - os << "hipJitOptionInfoLogBufferSizeBytes"; - break; - case hipJitOptionErrorLogBuffer: - os << "hipJitOptionErrorLogBuffer"; - break; - case hipJitOptionErrorLogBufferSizeBytes: - os << "hipJitOptionErrorLogBufferSizeBytes"; - break; - case hipJitOptionOptimizationLevel: - os << "hipJitOptionOptimizationLevel"; - break; - case hipJitOptionTargetFromContext: - os << "hipJitOptionTargetFromContext"; - break; - case hipJitOptionTarget: - os << "hipJitOptionTarget"; - break; - case hipJitOptionFallbackStrategy: - os << "hipJitOptionFallbackStrategy"; - break; - case hipJitOptionGenerateDebugInfo: - os << "hipJitOptionGenerateDebugInfo"; - break; - case hipJitOptionCacheMode: - os << "hipJitOptionCacheMode"; - break; - case hipJitOptionSm3xOpt: - os << "hipJitOptionSm3xOpt"; - break; - case hipJitOptionFastCompile: - os << "hipJitOptionFastCompile"; - break; - case hipJitOptionNumOptions: - os << "hipJitOptionNumOptions"; - break; - default: - os << "hipJitOptionMaxRegisters"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipFuncCache_t& s) { - switch (s) { - case hipFuncCachePreferNone: - os << "hipFuncCachePreferNone"; - break; - case hipFuncCachePreferShared: - os << "hipFuncCachePreferShared"; - break; - case hipFuncCachePreferL1: - os << "hipFuncCachePreferL1"; - break; - case hipFuncCachePreferEqual: - os << "hipFuncCachePreferEqual"; - break; - default: - os << "hipFuncCachePreferNone"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipSharedMemConfig& s) { - switch (s) { - case hipSharedMemBankSizeDefault: - os << "hipSharedMemBankSizeDefault"; - break; - case hipSharedMemBankSizeFourByte: - os << "hipSharedMemBankSizeFourByte"; - break; - case hipSharedMemBankSizeEightByte: - os << "hipSharedMemBankSizeEightByte"; - break; - default: - os << "hipSharedMemBankSizeDefault"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipDataType& s) { - switch (s) { - case HIP_R_16F: - os << "HIP_R_16F"; - break; - case HIP_R_32F: - os << "HIP_R_32F"; - break; - case HIP_R_64F: - os << "HIP_R_64F"; - break; - case HIP_C_16F: - os << "HIP_C_16F"; - break; - case HIP_C_32F: - os << "HIP_C_32F"; - break; - case HIP_C_64F: - os << "HIP_C_64F"; - break; - default: - os << "HIP_R_16F"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipLibraryPropertyType& s) { - switch (s) { - case HIP_LIBRARY_MAJOR_VERSION: - os << "HIP_LIBRARY_MAJOR_VERSION"; - break; - case HIP_LIBRARY_MINOR_VERSION: - os << "HIP_LIBRARY_MINOR_VERSION"; - break; - case HIP_LIBRARY_PATCH_LEVEL: - os << "HIP_LIBRARY_PATCH_LEVEL"; - break; - default: - os << "HIP_LIBRARY_MAJOR_VERSION"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hip_api_id_t& s) { - os << hip_api_name(s); - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hip_api_id_t* s) { - if (s) { - os << *s; - } else { - os << "nullptr"; - } - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipTextureDesc& s) { - os << '{' - << '{' - << s.addressMode[0] - << ',' - << s.addressMode[1] - << ',' - << s.addressMode[2] - << '}' - << ',' - << s.filterMode - << ',' - << s.readMode - << ',' - << s.sRGB - << ',' - << '{' - << s.borderColor[0] - << ',' - << s.borderColor[1] - << ',' - << s.borderColor[2] - << ',' - << s.borderColor[3] - << '}' - << ',' - << s.normalizedCoords - << ',' - << s.mipmapFilterMode - << ',' - << s.mipmapLevelBias - << ',' - << s.minMipmapLevelClamp - << ',' - << s.maxMipmapLevelClamp - << '}'; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipTextureDesc* s) { - if (s) { - os << *s; - } else { - os << "nullptr"; - } - return os; -} - - -inline std::ostream& operator<<(std::ostream& os, const dim3& s) { - os << '{' - << s.x - << ',' - << s.y - << ',' - << s.z - << '}'; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const dim3* s) { - if (s) { - os << *s; - } else { - os << "nullptr"; - } - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatDesc& s) { - os << '{' - << s.x - << ',' - << s.y - << ',' - << s.z - << ',' - << s.w - << ',' - << s.f - << '}'; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatDesc* s) { - if (s) { - os << *s; - } else { - os << "nullptr"; - } - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipMipmappedArray& s) { - os << '{' - << s.data - << ',' - << s.desc - << ',' - << s.width - << ',' - << s.height - << ',' - << s.depth - << '}'; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const hipMipmappedArray* s) { - if (s) { - os << *s; - } else { - os << "nullptr"; - } - return os; -} - - -inline std::ostream& operator<<(std::ostream& os, const hipResourceDesc& s) { - os << '{' - << s.resType - << ',' - << '{'; - - switch (s.resType) { - case hipResourceTypeLinear: - os << s.res.linear.devPtr - << ',' - << s.res.linear.desc - << ',' - << s.res.linear.sizeInBytes; - break; - case hipResourceTypePitch2D: - os << s.res.pitch2D.devPtr - << ',' - << s.res.pitch2D.desc - << ',' - << s.res.pitch2D.width - << ',' - << s.res.pitch2D.height - << ',' - << s.res.pitch2D.pitchInBytes; - break; - case hipResourceTypeArray: - os << s.res.array.array; - break; - case hipResourceTypeMipmappedArray: - os <(hmod)); - device::Program* dev_program = program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]); - if (dev_program == nullptr) { - LogPrintfError("Cannot get Device Program for module: 0x%x \n", hmod); - guarantee(false, "Cannot get Device Program"); - } - - if(!dev_program->createGlobalVarObj(&amd_mem_obj_, &device_ptr_, &size_, name.c_str())) { - LogPrintfError("Cannot create Global Var obj for symbol: %s \n", name.c_str()); - guarantee(false, "Cannot create GlobalVar Obj"); - } - - // Handle size 0 symbols - if (size_ != 0) { - if (amd_mem_obj_ == nullptr || device_ptr_ == nullptr) { - LogPrintfError("Cannot get memory for creating device Var: %s", name.c_str()); - guarantee(false, "Cannot get memory for creating device var"); - } - amd::MemObjMap::AddMemObj(device_ptr_, amd_mem_obj_); - } -} - -DeviceVar::~DeviceVar() { - if (amd_mem_obj_ != nullptr) { - amd::MemObjMap::RemoveMemObj(device_ptr_); - amd_mem_obj_->release(); - } - - if (shadowVptr != nullptr) { - textureReference* texRef = reinterpret_cast(shadowVptr); - delete texRef; - shadowVptr = nullptr; - } - - device_ptr_ = nullptr; - size_ = 0; -} - -//Device Functions -DeviceFunc::DeviceFunc(std::string name, hipModule_t hmod) : dflock_("function lock"), - name_(name), kernel_(nullptr) { - amd::Program* program = as_amd(reinterpret_cast(hmod)); - - const amd::Symbol *symbol = program->findSymbol(name.c_str()); - if (symbol == nullptr) { - LogPrintfError("Cannot find Symbol with name: %s \n", name.c_str()); - guarantee(false, "Cannot find Symbol"); - } - - kernel_ = new amd::Kernel(*program, *symbol, name); - if (kernel_ == nullptr) { - LogPrintfError("Cannot create kernel with name: %s \n", name.c_str()); - guarantee(false, "Cannot Create kernel"); - } -} - -DeviceFunc::~DeviceFunc() { - if (kernel_ != nullptr) { - kernel_->release(); - } -} - -//Abstract functions -Function::Function(std::string name, FatBinaryInfo** modules) - : name_(name), modules_(modules) { - dFunc_.resize(g_devices.size()); -} - -Function::~Function() { - for (auto& elem : dFunc_) { - delete elem; - } - name_ = ""; - modules_ = nullptr; -} - -hipError_t Function::getDynFunc(hipFunction_t* hfunc, hipModule_t hmod) { - guarantee((dFunc_.size() == g_devices.size()), "dFunc Size mismatch"); - if (dFunc_[ihipGetDevice()] == nullptr) { - dFunc_[ihipGetDevice()] = new DeviceFunc(name_, hmod); - } - *hfunc = dFunc_[ihipGetDevice()]->asHipFunction(); - - return hipSuccess; -} - -hipError_t Function::getStatFunc(hipFunction_t* hfunc, int deviceId) { - guarantee(modules_ != nullptr, "Module not initialized"); - - hipModule_t hmod = nullptr; - IHIP_RETURN_ONFAIL((*modules_)->BuildProgram(deviceId)); - IHIP_RETURN_ONFAIL((*modules_)->GetModule(deviceId, &hmod)); - - if (dFunc_[deviceId] == nullptr) { - dFunc_[deviceId] = new DeviceFunc(name_, hmod); - } - *hfunc = dFunc_[deviceId]->asHipFunction(); - - return hipSuccess; -} - -hipError_t Function::getStatFuncAttr(hipFuncAttributes* func_attr, int deviceId) { - guarantee((modules_ != nullptr), "Module not initialized"); - - hipModule_t hmod = nullptr; - IHIP_RETURN_ONFAIL((*modules_)->BuildProgram(deviceId)); - IHIP_RETURN_ONFAIL((*modules_)->GetModule(deviceId, &hmod)); - - if (dFunc_[deviceId] == nullptr) { - dFunc_[deviceId] = new DeviceFunc(name_, hmod); - } - - const std::vector& devices = amd::Device::getDevices(CL_DEVICE_TYPE_GPU, false); - - amd::Kernel* kernel = dFunc_[deviceId]->kernel(); - const device::Kernel::WorkGroupInfo* wginfo = kernel->getDeviceKernel(*devices[deviceId])->workGroupInfo(); - func_attr->sharedSizeBytes = static_cast(wginfo->localMemSize_); - func_attr->binaryVersion = static_cast(kernel->signature().version()); - func_attr->cacheModeCA = 0; - func_attr->constSizeBytes = 0; - func_attr->localSizeBytes = wginfo->privateMemSize_; - func_attr->maxDynamicSharedSizeBytes = static_cast(wginfo->availableLDSSize_ - - wginfo->localMemSize_); - - func_attr->maxThreadsPerBlock = static_cast(wginfo->size_); - func_attr->numRegs = static_cast(wginfo->usedVGPRs_); - func_attr->preferredShmemCarveout = 0; - func_attr->ptxVersion = 30; - - - return hipSuccess; -} - -//Abstract Vars -Var::Var(std::string name, DeviceVarKind dVarKind, size_t size, int type, int norm, - FatBinaryInfo** modules) : name_(name), dVarKind_(dVarKind), size_(size), - type_(type), norm_(norm), modules_(modules) { - dVar_.resize(g_devices.size()); -} - -Var::Var(std::string name, DeviceVarKind dVarKind, void *pointer, size_t size, - unsigned align, FatBinaryInfo** modules) : name_(name), dVarKind_(dVarKind), - size_(size), modules_(modules), managedVarPtr_(pointer), align_(align) { - dVar_.resize(g_devices.size()); -} - -Var::~Var() { - for (auto& elem : dVar_) { - delete elem; - } - modules_ = nullptr; -} - -hipError_t Var::getDeviceVar(DeviceVar** dvar, int deviceId, hipModule_t hmod) { - guarantee((deviceId >= 0), "Invalid DeviceId, less than zero"); - guarantee((static_cast(deviceId) < g_devices.size()), - "Invalid DeviceId, greater than no of code objects"); - guarantee((dVar_.size() == g_devices.size()), - "Device Var not initialized to size"); - - if (dVar_[deviceId] == nullptr) { - dVar_[deviceId] = new DeviceVar(name_, hmod); - } - - *dvar = dVar_[deviceId]; - return hipSuccess; -} - -hipError_t Var::getStatDeviceVar(DeviceVar** dvar, int deviceId) { - guarantee((deviceId >= 0) , "Invalid DeviceId, less than zero"); - guarantee((static_cast(deviceId) < g_devices.size()), - "Invalid DeviceId, greater than no of code objects"); - if (dVar_[deviceId] == nullptr) { - hipModule_t hmod = nullptr; - IHIP_RETURN_ONFAIL((*modules_)->BuildProgram(deviceId)); - IHIP_RETURN_ONFAIL((*modules_)->GetModule(deviceId, &hmod)); - dVar_[deviceId] = new DeviceVar(name_, hmod); - } - *dvar = dVar_[deviceId]; - return hipSuccess; -} - -}; //namespace: hip diff --git a/src/hipamd/src/hip_global.hpp b/src/hipamd/src/hip_global.hpp deleted file mode 100755 index 48ea0c03ce..0000000000 --- a/src/hipamd/src/hip_global.hpp +++ /dev/null @@ -1,127 +0,0 @@ -#ifndef HIP_GLOBAL_HPP -#define HIP_GLOBAL_HPP - -#include -#include - -#include "hip/hip_runtime_api.h" -#include "hip/hip_runtime.h" -#include "hip_internal.hpp" -#include "hip_fatbin.hpp" -#include "platform/program.hpp" - -namespace hip { - -//Forward Declaration -class CodeObject; - -//Device Structures -class DeviceVar { -public: - DeviceVar(std::string name, hipModule_t hmod); - ~DeviceVar(); - - //Accessors for device ptr and size, populated during constructor. - hipDeviceptr_t device_ptr() const { return device_ptr_; } - size_t size() const { return size_; } - std::string name() const { return name_; } - void* shadowVptr; - -private: - std::string name_; //Name of the var - amd::Memory* amd_mem_obj_; //amd_mem_obj abstraction - hipDeviceptr_t device_ptr_; //Device Pointer - size_t size_; //Size of the var -}; - -class DeviceFunc { -public: - DeviceFunc(std::string name, hipModule_t hmod); - ~DeviceFunc(); - - amd::Monitor dflock_; - - //Converts DeviceFunc to hipFunction_t(used by app) and vice versa. - hipFunction_t asHipFunction() { return reinterpret_cast(this); } - static DeviceFunc* asFunction(hipFunction_t f) { return reinterpret_cast(f); } - - //Accessor for kernel_ and name_ populated during constructor. - std::string name() const { return name_; } - amd::Kernel* kernel() const { return kernel_; } - -private: - std::string name_; //name of the func(not unique identifier) - amd::Kernel* kernel_; //Kernel ptr referencing to ROCclr Symbol -}; - -//Abstract Structures -class Function { -public: - Function(std::string name, FatBinaryInfo** modules=nullptr); - ~Function(); - - //Return DeviceFunc for this this dynamically loaded module - hipError_t getDynFunc(hipFunction_t* hfunc, hipModule_t hmod); - - //Return Device Func & attr . Generate/build if not already done so. - hipError_t getStatFunc(hipFunction_t *hfunc, int deviceId); - hipError_t getStatFuncAttr(hipFuncAttributes* func_attr, int deviceId); - void resize_dFunc(size_t size) { dFunc_.resize(size); } - FatBinaryInfo** moduleInfo() { return modules_; }; - -private: - std::vector dFunc_; //DeviceFuncObj per Device - std::string name_; //name of the func(not unique identifier) - FatBinaryInfo** modules_; // static module where it is referenced -}; - -class Var { -public: - //Types of variable - enum DeviceVarKind { - DVK_Variable = 0, - DVK_Surface, - DVK_Texture, - DVK_Managed - }; - - Var(std::string name, DeviceVarKind dVarKind, size_t size, int type, int norm, - FatBinaryInfo** modules = nullptr); - - Var(std::string name, DeviceVarKind dVarKind, void *pointer, size_t size, unsigned align, - FatBinaryInfo** modules = nullptr); - - ~Var(); - - //Return DeviceVar for this dynamically loaded module - hipError_t getDeviceVar(DeviceVar** dvar, int deviceId, hipModule_t hmod); - - //Return DeviceVar for module Generate/build if not already done so. - hipError_t getStatDeviceVar(DeviceVar** dvar, int deviceId); - void resize_dVar(size_t size) { dVar_.resize(size); } - - FatBinaryInfo** moduleInfo() { return modules_; }; - DeviceVarKind getVarKind() const { return dVarKind_; } - size_t getSize() const { return size_; } - - void* getManagedVarPtr() { return managedVarPtr_; }; - void setManagedVarInfo(void* pointer, size_t size) { - managedVarPtr_ = pointer; - size_ = size; - dVarKind_ = DVK_Managed; - } -private: - std::vector dVar_; // DeviceVarObj per Device - std::string name_; // Variable name (not unique identifier) - DeviceVarKind dVarKind_; // Variable kind - size_t size_; // Size of the variable - int type_; // Type(Textures/Surfaces only) - int norm_; // Type(Textures/Surfaces only) - FatBinaryInfo** modules_; // static module where it is referenced - - void *managedVarPtr_; // Managed memory pointer with size_ & align_ - unsigned int align_; // Managed memory alignment -}; - -}; //namespace: hip -#endif /* HIP_GLOBAL_HPP */ diff --git a/src/hipamd/src/hip_graph.cpp b/src/hipamd/src/hip_graph.cpp deleted file mode 100644 index a8f5df5100..0000000000 --- a/src/hipamd/src/hip_graph.cpp +++ /dev/null @@ -1,415 +0,0 @@ -/* Copyright (c) 2021-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "hip_graph_internal.hpp" -#include "platform/command.hpp" -#include "hip_conversions.hpp" -#include "hip_platform.hpp" -#include "hip_event.hpp" - -thread_local std::vector g_captureStreams; -std::unordered_map hipGraphExec::activeGraphExec_; - -hipError_t ihipGraphAddKernelNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, - const hipGraphNode_t* pDependencies, size_t numDependencies, - const hipKernelNodeParams* pNodeParams) { - if (pGraphNode == nullptr || graph == nullptr || - (numDependencies > 0 && pDependencies == nullptr) || pNodeParams == nullptr) { - return hipErrorInvalidValue; - } - hipFunction_t func = nullptr; - hipError_t status = - PlatformState::instance().getStatFunc(&func, pNodeParams->func, ihipGetDevice()); - if ((status != hipSuccess) || (func == nullptr)) { - return hipErrorInvalidDeviceFunction; - } - size_t globalWorkSizeX = static_cast(pNodeParams->gridDim.x) * pNodeParams->blockDim.x; - size_t globalWorkSizeY = static_cast(pNodeParams->gridDim.y) * pNodeParams->blockDim.y; - size_t globalWorkSizeZ = static_cast(pNodeParams->gridDim.z) * pNodeParams->blockDim.z; - if (globalWorkSizeX > std::numeric_limits::max() || - globalWorkSizeY > std::numeric_limits::max() || - globalWorkSizeZ > std::numeric_limits::max()) { - return hipErrorInvalidConfiguration; - } - status = ihipLaunchKernel_validate( - func, static_cast(globalWorkSizeX), static_cast(globalWorkSizeY), - static_cast(globalWorkSizeZ), pNodeParams->blockDim.x, pNodeParams->blockDim.y, - pNodeParams->blockDim.z, pNodeParams->sharedMemBytes, pNodeParams->kernelParams, - pNodeParams->extra, ihipGetDevice(), 0); - if (status != hipSuccess) { - return status; - } - *pGraphNode = new hipGraphKernelNode(pNodeParams, func); - if (numDependencies == 0) { - graph->AddNode(*pGraphNode); - } - for (size_t i = 0; i < numDependencies; i++) { - if (graph->AddEdge(*(pDependencies + i), *pGraphNode) != hipSuccess) { - return hipErrorInvalidValue; - } - } - return hipSuccess; -} - -hipError_t ihipGraphAddMemcpyNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, - const hipGraphNode_t* pDependencies, size_t numDependencies, - const hipMemcpy3DParms* pCopyParams) { - if (pGraphNode == nullptr || graph == nullptr || - (numDependencies > 0 && pDependencies == nullptr) || pCopyParams == nullptr) { - return hipErrorInvalidValue; - } - ihipMemcpy3D_validate(pCopyParams); - *pGraphNode = new hipGraphMemcpyNode(pCopyParams); - if (numDependencies == 0) { - graph->AddNode(*pGraphNode); - } - for (size_t i = 0; i < numDependencies; i++) { - if (graph->AddEdge(*(pDependencies + i), *pGraphNode) != hipSuccess) { - return hipErrorInvalidValue; - } - } - return hipSuccess; -} - -hipError_t ihipGraphAddMemsetNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, - const hipGraphNode_t* pDependencies, size_t numDependencies, - const hipMemsetParams* pMemsetParams) { - if (pGraphNode == nullptr || graph == nullptr || - (numDependencies > 0 && pDependencies == nullptr) || pMemsetParams == nullptr) { - return hipErrorInvalidValue; - } - if (pMemsetParams->height == 1) { - ihipMemset_validate(pMemsetParams->dst, pMemsetParams->value, pMemsetParams->elementSize, - pMemsetParams->width * pMemsetParams->elementSize); - } else { - auto sizeBytes = pMemsetParams->width * pMemsetParams->height * 1; - ihipMemset3D_validate( - {pMemsetParams->dst, pMemsetParams->pitch, pMemsetParams->width, pMemsetParams->height}, - pMemsetParams->value, {pMemsetParams->width, pMemsetParams->height, 1}, sizeBytes); - } - - *pGraphNode = new hipGraphMemsetNode(pMemsetParams); - if (numDependencies == 0) { - graph->AddNode(*pGraphNode); - } - for (size_t i = 0; i < numDependencies; i++) { - if (graph->AddEdge(*(pDependencies + i), *pGraphNode) != hipSuccess) { - return hipErrorInvalidValue; - } - } - return hipSuccess; -} - -hipError_t capturehipLaunchKernel(hipStream_t& stream, const void*& hostFunction, dim3& gridDim, - dim3& blockDim, void**& args, size_t& sharedMemBytes) { - ClPrint(amd::LOG_INFO, amd::LOG_API, - "[hipGraph] current capture node kernel launch on stream : %p", stream); - hip::Stream* s = reinterpret_cast(stream); - hipKernelNodeParams nodeParams; - nodeParams.func = const_cast(hostFunction); - nodeParams.blockDim = blockDim; - nodeParams.extra = nullptr; - nodeParams.gridDim = gridDim; - nodeParams.kernelParams = args; - nodeParams.sharedMemBytes = sharedMemBytes; - - hipGraphNode_t pGraphNode; - hipError_t status = - ihipGraphAddKernelNode(&pGraphNode, s->GetCaptureGraph(), s->GetLastCapturedNodes().data(), - s->GetLastCapturedNodes().size(), &nodeParams); - if (status != hipSuccess) { - return status; - } - s->SetLastCapturedNode(pGraphNode); - return hipSuccess; -} - -hipError_t capturehipMemcpy3DAsync(hipStream_t& stream, const hipMemcpy3DParms*& p) { - ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memcpy3D on stream : %p", - stream); - hip::Stream* s = reinterpret_cast(stream); - hipGraphNode_t pGraphNode; - hipError_t status = - ihipGraphAddMemcpyNode(&pGraphNode, s->GetCaptureGraph(), s->GetLastCapturedNodes().data(), - s->GetLastCapturedNodes().size(), p); - if (status != hipSuccess) { - return status; - } - s->SetLastCapturedNode(pGraphNode); - return hipSuccess; -} - -hipError_t capturehipMemcpyAsync(hipStream_t& stream, void*& dst, const void*& src, - size_t& sizeBytes, hipMemcpyKind& kind) { - ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memcpy1D on stream : %p", - stream); - hip::Stream* s = reinterpret_cast(stream); - hipGraphNode_t pGraphNode; - hipGraph_t graph = nullptr; - std::vector pDependencies = s->GetLastCapturedNodes(); - size_t numDependencies = s->GetLastCapturedNodes().size(); - graph = s->GetCaptureGraph(); - ihipMemcpy_validate(dst, src, sizeBytes, kind); - pGraphNode = new hipGraphMemcpyNode1D(dst, src, sizeBytes, kind); - if (numDependencies == 0) { - graph->AddNode(pGraphNode); - } - for (size_t i = 0; i < numDependencies; i++) { - if (graph->AddEdge(pDependencies[i], pGraphNode) != hipSuccess) { - return hipErrorInvalidValue; - } - } - s->SetLastCapturedNode(pGraphNode); - return hipSuccess; -} - -hipError_t capturehipMemcpyFromSymbolAsync(hipStream_t& stream, void*& dst, const void*& symbol, - size_t& sizeBytes, size_t& offset, hipMemcpyKind& kind) { - ClPrint(amd::LOG_INFO, amd::LOG_API, - "[hipGraph] current capture node MemcpyFromSymbolNode on stream : %p", stream); - return hipSuccess; -} - -hipError_t capturehipMemcpyToSymbolAsync(hipStream_t& stream, const void*& symbol, const void*& src, - size_t& sizeBytes, size_t& offset, hipMemcpyKind& kind) { - ClPrint(amd::LOG_INFO, amd::LOG_API, - "[hipGraph] current capture node MemcpyToSymbolNode on stream : %p", stream); - return hipSuccess; -} - -hipError_t capturehipMemsetAsync(hipStream_t& stream, void*& dst, int& value, size_t& valueSize, - size_t& sizeBytes) { - ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memset1D on stream : %p", - stream); - - hipMemsetParams memsetParams = {0}; - memsetParams.dst = dst; - memsetParams.value = value; - memsetParams.elementSize = valueSize; - memsetParams.width = sizeBytes / valueSize; - memsetParams.height = 1; - - hip::Stream* s = reinterpret_cast(stream); - hipGraphNode_t pGraphNode; - hipError_t status = - ihipGraphAddMemsetNode(&pGraphNode, s->GetCaptureGraph(), s->GetLastCapturedNodes().data(), - s->GetLastCapturedNodes().size(), &memsetParams); - if (status != hipSuccess) { - return status; - } - s->SetLastCapturedNode(pGraphNode); - return hipSuccess; -} - -hipError_t capturehipMemset2DAsync(hipStream_t& stream, void*& dst, size_t& pitch, int& value, - size_t& width, size_t& height) { - ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memset2D on stream : %p", - stream); - hipMemsetParams memsetParams = {0}; - - memsetParams.dst = dst; - memsetParams.value = value; - memsetParams.width = width; - memsetParams.height = height; - memsetParams.pitch = pitch; - hip::Stream* s = reinterpret_cast(stream); - hipGraphNode_t pGraphNode; - hipError_t status = - ihipGraphAddMemsetNode(&pGraphNode, s->GetCaptureGraph(), s->GetLastCapturedNodes().data(), - s->GetLastCapturedNodes().size(), &memsetParams); - if (status != hipSuccess) { - return status; - } - s->SetLastCapturedNode(pGraphNode); - return hipSuccess; -} - -hipError_t capturehipMemset3DAsync(hipStream_t& stream, hipPitchedPtr& pitchedDevPtr, int& value, - hipExtent& extent) { - ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memset3D on stream : %p", - stream); - return hipSuccess; -} - -hipError_t capturehipEventRecord(hipStream_t& stream, hipEvent_t& event) { - ClPrint(amd::LOG_INFO, amd::LOG_API, - "[hipGraph] current capture node EventRecord on stream : %p, Event %p", stream, event); - if (event == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - hip::Event* e = reinterpret_cast(event); - e->StartCapture(stream); - hip::Stream* s = reinterpret_cast(stream); - std::vector lastCapturedNodes = s->GetLastCapturedNodes(); - if (!lastCapturedNodes.empty()) { - e->SetNodesPrevToRecorded(lastCapturedNodes); - } - return hipSuccess; -} - -hipError_t capturehipStreamWaitEvent(hipEvent_t& event, hipStream_t& stream, unsigned int& flags) { - ClPrint(amd::LOG_INFO, amd::LOG_API, - "[hipGraph] current capture node StreamWaitEvent on stream : %p, Event %p", stream, - event); - hip::Stream* s = reinterpret_cast(stream); - hip::Event* e = reinterpret_cast(event); - - if (event == nullptr || stream == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - if (!s->IsOriginStream()) { - s->SetCaptureGraph(reinterpret_cast(e->GetCaptureStream())->GetCaptureGraph()); - s->SetCaptureMode(reinterpret_cast(e->GetCaptureStream())->GetCaptureMode()); - s->SetParentStream(e->GetCaptureStream()); - } - s->AddCrossCapturedNode(e->GetNodesPrevToRecorded()); - g_captureStreams.push_back(stream); - return hipSuccess; -} - -hipError_t hipStreamIsCapturing(hipStream_t stream, hipStreamCaptureStatus** pCaptureStatus) { - HIP_INIT_API(hipStreamIsCapturing, stream, pCaptureStatus); - if (stream == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - hipStreamCaptureStatus captureStatus = reinterpret_cast(stream)->GetCaptureStatus(); - *pCaptureStatus = &captureStatus; - HIP_RETURN(hipSuccess); -} - -hipError_t hipStreamBeginCapture(hipStream_t stream, hipStreamCaptureMode mode) { - HIP_INIT_API(hipStreamBeginCapture, stream, mode); - hip::Stream* s = reinterpret_cast(stream); - // capture cannot be initiated on legacy stream - // It can be initiated if the stream is not already in capture mode - if (stream == nullptr || s->GetCaptureStatus() == hipStreamCaptureStatusActive) { - HIP_RETURN(hipErrorInvalidValue); - } - s->SetCaptureGraph(new hipGraph()); - s->SetCaptureMode(mode); - s->SetOriginStream(); - g_captureStreams.push_back(stream); - HIP_RETURN_DURATION(hipSuccess); -} - -hipError_t hipStreamEndCapture(hipStream_t stream, hipGraph_t* pGraph) { - HIP_INIT_API(hipStreamEndCapture, stream, pGraph); - hip::Stream* s = reinterpret_cast(stream); - // Capture must be ended on the same stream in which it was initiated - if (!s->IsOriginStream()) { - HIP_RETURN(hipErrorStreamCaptureUnmatched); - } - // If mode is not hipStreamCaptureModeRelaxed, hipStreamEndCapture must be called on the stream - // from the same thread - if (s->GetCaptureMode() != hipStreamCaptureModeRelaxed && - std::find(g_captureStreams.begin(), g_captureStreams.end(), stream) == - g_captureStreams.end()) { - HIP_RETURN(hipErrorStreamCaptureWrongThread); - } - // If capture was invalidated, due to a violation of the rules of stream capture - if (s->GetCaptureStatus() == hipStreamCaptureStatusInvalidated) { - *pGraph = nullptr; - HIP_RETURN(hipErrorStreamCaptureInvalidated); - } - // check if all parallel streams have joined - if (s->GetCaptureGraph()->GetLeafNodeCount() != 1) { - return hipErrorStreamCaptureUnjoined; - } - *pGraph = s->GetCaptureGraph(); - // end capture on all streams/events part of graph capture - HIP_RETURN_DURATION(s->EndCapture()); -} - -hipError_t hipGraphCreate(hipGraph_t* pGraph, unsigned int flags) { - HIP_INIT_API(hipGraphCreate, pGraph, flags); - *pGraph = new hipGraph(); - HIP_RETURN(hipSuccess); -} - -hipError_t hipGraphDestroy(hipGraph_t graph) { - HIP_INIT_API(hipGraphDestroy, graph); - delete graph; - HIP_RETURN(hipSuccess); -} - -hipError_t hipGraphAddKernelNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, - const hipGraphNode_t* pDependencies, size_t numDependencies, - const hipKernelNodeParams* pNodeParams) { - HIP_INIT_API(hipGraphAddKernelNode, pGraphNode, graph, pDependencies, numDependencies, - pNodeParams); - HIP_RETURN_DURATION( - ihipGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams);); -} - -hipError_t hipGraphAddMemcpyNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, - const hipGraphNode_t* pDependencies, size_t numDependencies, - const hipMemcpy3DParms* pCopyParams) { - HIP_INIT_API(hipGraphAddMemcpyNode, pGraphNode, graph, pDependencies, numDependencies, - pCopyParams); - - HIP_RETURN_DURATION( - ihipGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams);); -} - -hipError_t hipGraphAddMemsetNode(hipGraphNode_t* pGraphNode, hipGraph_t graph, - const hipGraphNode_t* pDependencies, size_t numDependencies, - const hipMemsetParams* pMemsetParams) { - HIP_INIT_API(hipGraphAddMemsetNode, pGraphNode, graph, pDependencies, numDependencies, - pMemsetParams); - - HIP_RETURN_DURATION( - ihipGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams);); -} - -hipError_t ihipGraphInstantiate(hipGraphExec_t* pGraphExec, hipGraph_t graph, - hipGraphNode_t* pErrorNode, char* pLogBuffer, size_t bufferSize) { - std::vector> parallelLists; - std::unordered_map> nodeWaitLists; - graph->GetRunList(parallelLists, nodeWaitLists); - std::vector levelOrder; - graph->LevelOrder(levelOrder); - *pGraphExec = new hipGraphExec(levelOrder, parallelLists, nodeWaitLists); - if (*pGraphExec != nullptr) { - return (*pGraphExec)->Init(); - } else { - return hipErrorOutOfMemory; - } -} - -hipError_t hipGraphInstantiate(hipGraphExec_t* pGraphExec, hipGraph_t graph, - hipGraphNode_t* pErrorNode, char* pLogBuffer, size_t bufferSize) { - HIP_INIT_API(hipGraphInstantiate, pGraphExec, graph); - HIP_RETURN_DURATION(ihipGraphInstantiate(pGraphExec, graph, pErrorNode, pLogBuffer, bufferSize)); -} - -hipError_t hipGraphExecDestroy(hipGraphExec_t pGraphExec) { - HIP_INIT_API(hipGraphExecDestroy, pGraphExec); - delete pGraphExec; - HIP_RETURN(hipSuccess); -} - -hipError_t ihipGraphlaunch(hipGraphExec_t graphExec, hipStream_t stream) { - return graphExec->Run(stream); -} - -hipError_t hipGraphLaunch(hipGraphExec_t graphExec, hipStream_t stream) { - HIP_INIT_API(hipGraphLaunch, graphExec, stream); - HIP_RETURN_DURATION(ihipGraphlaunch(graphExec, stream)); -} diff --git a/src/hipamd/src/hip_graph_capture.hpp b/src/hipamd/src/hip_graph_capture.hpp deleted file mode 100644 index 16abfdeef7..0000000000 --- a/src/hipamd/src/hip_graph_capture.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright (c) 2021-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#pragma once -// forward declaration of capture methods -hipError_t capturehipLaunchKernel(hipStream_t& stream, const void*& hostFunction, dim3& gridDim, - dim3& blockDim, void**& args, size_t& sharedMemBytes); - -hipError_t capturehipMemcpy3DAsync(hipStream_t& stream, const hipMemcpy3DParms*& p); - -hipError_t capturehipMemcpyAsync(hipStream_t& stream, void*& dst, const void*& src, - size_t& sizeBytes, hipMemcpyKind& kind); - -hipError_t capturehipMemcpyFromSymbolAsync(hipStream_t& stream, void*& dst, const void*& symbol, - size_t& sizeBytes, size_t& offset, hipMemcpyKind& kind); - -hipError_t capturehipMemcpyToSymbolAsync(hipStream_t& stream, const void*& symbol, const void*& src, - size_t& sizeBytes, size_t& offset, hipMemcpyKind& kind); - -hipError_t capturehipMemsetAsync(hipStream_t& stream, void*& dst, int& value, size_t& valueSize, - size_t& sizeBytes); - -hipError_t capturehipMemset2DAsync(hipStream_t& stream, void*& dst, size_t& pitch, int& value, - size_t& width, size_t& height); - -hipError_t capturehipMemset3DAsync(hipStream_t& stream, hipPitchedPtr& pitchedDevPtr, int& value, - hipExtent& extent); - -hipError_t capturehipEventRecord(hipStream_t& stream, hipEvent_t& event); - -hipError_t capturehipStreamWaitEvent(hipEvent_t& event, hipStream_t& stream, unsigned int& flags); diff --git a/src/hipamd/src/hip_graph_helper.hpp b/src/hipamd/src/hip_graph_helper.hpp deleted file mode 100644 index b28c14e811..0000000000 --- a/src/hipamd/src/hip_graph_helper.hpp +++ /dev/null @@ -1,35 +0,0 @@ -hipError_t ihipMemcpy3D_validate(const hipMemcpy3DParms* p); - -hipError_t ihipMemcpy_validate(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind); - -hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src, size_t sizeBytes, - hipMemcpyKind kind, amd::HostQueue& queue); - -hipError_t ihipLaunchKernel_validate(hipFunction_t f, uint32_t globalWorkSizeX, - uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, - uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ, - uint32_t sharedMemBytes, void** kernelParams, void** extra, - int deviceId, uint32_t params); - -hipError_t ihipMemset_validate(void* dst, int64_t value, size_t valueSize, size_t sizeBytes); - -hipError_t ihipMemset3D_validate(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent, - size_t sizeBytes); - -hipError_t ihipLaunchKernelCommand(amd::Command*& command, hipFunction_t f, - uint32_t globalWorkSizeX, uint32_t globalWorkSizeY, - uint32_t globalWorkSizeZ, uint32_t blockDimX, uint32_t blockDimY, - uint32_t blockDimZ, uint32_t sharedMemBytes, - amd::HostQueue* queue, void** kernelParams, void** extra, - hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags, - uint32_t params, uint32_t gridId, uint32_t numGrids, - uint64_t prevGridSum, uint64_t allGridSum, uint32_t firstDevice); - -hipError_t ihipMemcpy3DCommand(amd::Command*& command, const hipMemcpy3DParms* p, - amd::HostQueue* queue); - -hipError_t ihipMemsetCommand(std::vector& commands, void* dst, int64_t value, - size_t valueSize, size_t sizeBytes, amd::HostQueue* queue); - -hipError_t ihipMemset3DCommand(std::vector& commands, hipPitchedPtr pitchedDevPtr, - int value, hipExtent extent, amd::HostQueue* queue); diff --git a/src/hipamd/src/hip_graph_internal.cpp b/src/hipamd/src/hip_graph_internal.cpp deleted file mode 100644 index 75eb6f6726..0000000000 --- a/src/hipamd/src/hip_graph_internal.cpp +++ /dev/null @@ -1,364 +0,0 @@ -/* Copyright (c) 2021-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "hip_graph_internal.hpp" -#include - -#define CASE_STRING(X, C) \ - case X: \ - case_string = #C; \ - break; -const char* GetGraphNodeTypeString(uint32_t op) { - const char* case_string; - switch (static_cast(op)) { - CASE_STRING(hipGraphNodeTypeKernel, KernelNode) - CASE_STRING(hipGraphNodeTypeMemcpy, Memcpy3DNode) - CASE_STRING(hipGraphNodeTypeMemset, MemsetNode) - CASE_STRING(hipGraphNodeTypeHost, HostNode) - CASE_STRING(hipGraphNodeTypeGraph, GraphNode) - CASE_STRING(hipGraphNodeTypeEmpty, EmptyNode) - CASE_STRING(hipGraphNodeTypeWaitEvent, WaitEventNode) - CASE_STRING(hipGraphNodeTypeEventRecord, EventRecordNode) - CASE_STRING(hipGraphNodeTypeMemcpy1D, Memcpy1DNode) - CASE_STRING(hipGraphNodeTypeMemcpyFromSymbol, MemcpyFromSymbolNode) - CASE_STRING(hipGraphNodeTypeMemcpyToSymbol, MemcpyToSymbolNode) - default: - case_string = "Unknown node type"; - }; - return case_string; -}; - -hipError_t hipGraph::AddNode(const Node& node) { - vertices_.emplace_back(node); - nodeOutDegree_[node] = 0; - nodeInDegree_[node] = 0; - node->SetLevel(0); - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] Add %s(%p)\n", - GetGraphNodeTypeString(node->GetType()), node); - return hipSuccess; -} - -hipError_t hipGraph::AddEdge(const Node& parentNode, const Node& childNode) { - // if vertice doesn't exist, add it to the graph - if (std::find(vertices_.begin(), vertices_.end(), parentNode) == vertices_.end()) { - AddNode(parentNode); - } - if (std::find(vertices_.begin(), vertices_.end(), childNode) == vertices_.end()) { - AddNode(childNode); - } - // Check if edge already exists - auto connectedEdges = edges_.find(parentNode); - if (connectedEdges != edges_.end()) { - if (std::find(connectedEdges->second.begin(), connectedEdges->second.end(), childNode) != - connectedEdges->second.end()) { - return hipSuccess; - } - connectedEdges->second.emplace_back(childNode); - } else { - edges_[parentNode] = {childNode}; - } - nodeOutDegree_[parentNode]++; - nodeInDegree_[childNode]++; - childNode->SetLevel(std::max(childNode->GetLevel(), parentNode->GetLevel() + 1)); - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] Add edge btwn %s(%p) - %s(%p)\n", - GetGraphNodeTypeString(parentNode->GetType()), parentNode, - GetGraphNodeTypeString(childNode->GetType()), childNode); - return hipSuccess; -} - -// root nodes are all vertices with 0 in-degrees -std::vector hipGraph::GetRootNodes() const { - std::vector roots; - for (auto entry : vertices_) { - if (nodeInDegree_.at(entry) == 0) { - roots.push_back(entry); - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] root node: %s(%p)\n", - GetGraphNodeTypeString(entry->GetType()), entry); - } - } - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "\n"); - return roots; -} - -// leaf nodes are all vertices with 0 out-degrees -std::vector hipGraph::GetLeafNodes() const { - std::vector leafNodes; - for (auto entry : vertices_) { - if (nodeOutDegree_.at(entry) == 0) { - leafNodes.push_back(entry); - } - } - return leafNodes; -} - -size_t hipGraph::GetLeafNodeCount() const { - int numLeafNodes = 0; - for (auto entry : vertices_) { - if (nodeOutDegree_.at(entry) == 0) { - numLeafNodes++; - } - } - return numLeafNodes; -} - -std::vector> hipGraph::GetEdges() const { - std::vector> edges; - for (const auto& i : edges_) { - for (const auto& j : i.second) { - edges.push_back(std::make_pair(i.first, j)); - } - } - return edges; -} - -void hipGraph::GetRunListUtil(Node v, std::unordered_map& visited, - std::vector& singleList, - std::vector>& parallelLists, - std::unordered_map>& dependencies) { - // Mark the current node as visited. - visited[v] = true; - singleList.push_back(v); - // Recurse for all the vertices adjacent to this vertex - for (auto& adjNode : edges_[v]) { - if (!visited[adjNode]) { - // For the parallel list nodes add parent as the dependency - if (singleList.empty()) { - ClPrint(amd::LOG_INFO, amd::LOG_CODE, - "[hipGraph] For %s(%p)- add parent as dependency %s(%p)\n", - GetGraphNodeTypeString(adjNode->GetType()), adjNode, - GetGraphNodeTypeString(v->GetType()), v); - dependencies[adjNode].push_back(v); - } - GetRunListUtil(adjNode, visited, singleList, parallelLists, dependencies); - } else { - for (auto& list : parallelLists) { - // Merge singleList when adjNode matches with the first element of the list in existing - // lists - if (adjNode == list[0]) { - for (auto k = singleList.rbegin(); k != singleList.rend(); ++k) { - list.insert(list.begin(), *k); - } - singleList.erase(singleList.begin(), singleList.end()); - } - } - // If the list cannot be merged with the existing list add as dependancy - if (!singleList.empty()) { - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] For %s(%p)- add dependency %s(%p)\n", - GetGraphNodeTypeString(adjNode->GetType()), adjNode, - GetGraphNodeTypeString(v->GetType()), v); - dependencies[adjNode].push_back(v); - } - } - } - if (!singleList.empty()) { - parallelLists.push_back(singleList); - singleList.erase(singleList.begin(), singleList.end()); - } -} -// The function to do Topological Sort. -// It uses recursive GetRunListUtil() -void hipGraph::GetRunList(std::vector>& parallelLists, - std::unordered_map>& dependencies) { - std::vector singleList; - - // Mark all the vertices as not visited - std::unordered_map visited; - for (auto node : vertices_) visited[node] = false; - - // Call the recursive helper function for all vertices one by one - for (auto node : vertices_) { - if (visited[node] == false) { - GetRunListUtil(node, visited, singleList, parallelLists, dependencies); - } - } - for (size_t i = 0; i < parallelLists.size(); i++) { - for (size_t j = 0; j < parallelLists[i].size(); j++) { - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] list %d - %s(%p)\n", i + 1, - GetGraphNodeTypeString(parallelLists[i][j]->GetType()), parallelLists[i][j]); - } - } -} - -hipError_t hipGraph::LevelOrder(std::vector& levelOrder) { - std::vector roots = GetRootNodes(); - std::unordered_map visited; - std::queue q; - for (auto it = roots.begin(); it != roots.end(); it++) { - q.push(*it); - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] %s(%p) level:%d \n", - GetGraphNodeTypeString((*it)->GetType()), *it, (*it)->GetLevel()); - } - while (!q.empty()) { - Node& node = q.front(); - q.pop(); - levelOrder.push_back(node); - for (const auto& i : edges_[node]) { - if (visited.find(i) == visited.end() && i->GetLevel() == (node->GetLevel() + 1)) { - q.push(i); - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] %s(%p) level:%d \n", - GetGraphNodeTypeString(i->GetType()), i, i->GetLevel()); - visited[i] = true; - } - } - } - return hipSuccess; -} - -hipError_t hipGraphExec::CreateQueues() { - parallelQueues_.reserve(parallelLists_.size()); - for (size_t i = 0; i < parallelLists_.size(); i++) { - amd::HostQueue* queue; - cl_command_queue_properties properties = - (callbacks_table.is_enabled() || HIP_FORCE_QUEUE_PROFILING) ? CL_QUEUE_PROFILING_ENABLE : 0; - queue = new amd::HostQueue(*hip::getCurrentDevice()->asContext(), - *hip::getCurrentDevice()->devices()[0], properties); - - bool result = (queue != nullptr) ? queue->create() : false; - // Create a host queue - if (result) { - parallelQueues_.push_back(queue); - } else { - ClPrint(amd::LOG_ERROR, amd::LOG_CODE, "[hipGraph] Failed to create host queue\n"); - return hipErrorOutOfMemory; - } - } - return hipSuccess; -} - -hipError_t hipGraphExec::FillCommands() { - // Create commands - int i = 0; - hipError_t status; - for (const auto& list : parallelLists_) { - for (auto& node : list) { - status = node->CreateCommand(parallelQueues_[i]); - if (status != hipSuccess) return status; - } - i++; - } - // Add waitlists for all the commands - for (auto& node : levelOrder_) { - auto nodeWaitList = nodeWaitLists_.find(node); - if (nodeWaitList != nodeWaitLists_.end()) { - amd::Command::EventWaitList waitList; - for (auto depNode : nodeWaitList->second) { - for (auto command : depNode->GetCommands()) { - waitList.push_back(command); - } - } - for (auto command : nodeWaitList->first->GetCommands()) { - command->updateEventWaitList(waitList); - } - } - } - return status; -} - - -hipError_t hipGraphExec::Init() { - hipError_t status; - status = CreateQueues(); - if (status != hipSuccess) { - return status; - } - status = FillCommands(); - if (status != hipSuccess) { - return status; - } - rootCommand_ = nullptr; - /// stream should execute next command after graph finishes - /// Add marker to the stream that waits for all the last commands in parallel queues of graph - for (auto& singleList : parallelLists_) { - graphLastCmdWaitList_.push_back(singleList.back()->GetCommands().back()); - } - return status; -} - -void hipGraphExec::ResetGraph(cl_event event, cl_int command_exec_status, void* user_data) { - ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] Inside resetGraph!\n"); - hipGraphExec_t graphExec = - hipGraphExec::activeGraphExec_[reinterpret_cast(user_data)]; - if (graphExec != nullptr) { - for (auto& node : graphExec->levelOrder_) { - for (auto& command : node->GetCommands()) { - command->resetStatus(CL_INT_MAX); - } - } - graphExec->rootCommand_->resetStatus(CL_INT_MAX); - graphExec->bExecPending_.store(false); - } else { - ClPrint(amd::LOG_ERROR, amd::LOG_CODE, "[hipGraph] graphExec is nullptr during resetGraph!\n"); - } -} - -hipError_t hipGraphExec::UpdateGraphToWaitOnRoot() { - for (auto& singleList : parallelLists_) { - amd::Command::EventWaitList waitList; - waitList.push_back(rootCommand_); - if (!singleList.empty()) { - auto commands = singleList[0]->GetCommands(); - if (!commands.empty()) { - commands[0]->updateEventWaitList(waitList); - } - } - } - return hipSuccess; -} - -hipError_t hipGraphExec::Run(hipStream_t stream) { - if (bExecPending_.load() == true) { - ClPrint( - amd::LOG_INFO, amd::LOG_CODE, - "[hipGraph] Same graph launched while previous one is active, wait for it to finish!\n"); - lastEnqueuedGraphCmd_->awaitCompletion(); - } - amd::HostQueue* queue = hip::getQueue(stream); - if (queue == nullptr) { - return hipErrorInvalidResourceHandle; - } - if (rootCommand_ == nullptr || rootCommand_->queue() != queue) { - if (rootCommand_ != nullptr) { - rootCommand_->release(); - } - rootCommand_ = new amd::Marker(*queue, false, {}); - UpdateGraphToWaitOnRoot(); - } - rootCommand_->enqueue(); - for (auto& node : levelOrder_) { - for (auto& command : node->GetCommands()) { - command->enqueue(); - } - } - - amd::Command* command = new amd::Marker(*queue, false, graphLastCmdWaitList_); - if (command == nullptr) { - return hipErrorOutOfMemory; - } - amd::Event& event = command->event(); - if (!event.setCallback(CL_COMPLETE, hipGraphExec::ResetGraph, command)) { - return hipErrorInvalidHandle; - } - hipGraphExec::activeGraphExec_[command] = this; - lastEnqueuedGraphCmd_ = command; - bExecPending_.store(true); - command->enqueue(); - command->release(); - return hipSuccess; -} \ No newline at end of file diff --git a/src/hipamd/src/hip_graph_internal.hpp b/src/hipamd/src/hip_graph_internal.hpp deleted file mode 100644 index 8c034176f7..0000000000 --- a/src/hipamd/src/hip_graph_internal.hpp +++ /dev/null @@ -1,355 +0,0 @@ -/* Copyright (c) 2021-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#pragma once -#include -#include -#include -#include -#include -#include -#include - -#include "hip/hip_runtime.h" -#include "hip_internal.hpp" -#include "hip_graph_helper.hpp" - -typedef hipGraphNode* Node; - -class hipGraphNode { - protected: - uint32_t level_; - hipGraphNodeType type_; - std::vector commands_; - bool visited_; - - public: - hipGraphNode(hipGraphNodeType type) { - type_ = type; - level_ = 0; - visited_ = false; - } - virtual ~hipGraphNode() { - for (auto command : commands_) { - delete command; - } - } - virtual hipError_t CreateCommand(amd::HostQueue* queue) { return hipSuccess; } - std::vector& GetCommands() { return commands_; } - hipGraphNodeType GetType() { return type_; } - uint32_t GetLevel() { return level_; } - void SetLevel(uint32_t level) { level_ = level; } -}; - -class hipGraph { - std::unordered_map nodeInDegree_; // count of in coming edges for every vertex - std::unordered_map nodeOutDegree_; // count of outgoing edges for every vertex - std::vector vertices_; - std::unordered_map> edges_; - - public: - hipGraph() {} - ~hipGraph(){}; - /// add node to the graph - hipError_t AddNode(const Node& node); - /// add edge to the graph - hipError_t AddEdge(const Node& parentNode, const Node& childNode); - /// Returns root nodes, all vertices with 0 in-degrees - std::vector GetRootNodes() const; - /// Returns leaf nodes, all vertices with 0 out-degrees - std::vector GetLeafNodes() const; - /// Returns number of leaf nodes - size_t GetLeafNodeCount() const; - /// Returns total numbers of nodes in the graph - size_t GetNodeCount() const { return vertices_.size(); } - /// returns all the nodes in the graph - std::vector GetNodes() const { return vertices_; } - /// returns all the edges in the graph - std::vector> GetEdges() const; - void GetRunListUtil(Node v, std::unordered_map& visited, - std::vector& singleList, std::vector>& parallelList, - std::unordered_map>& dependencies); - void GetRunList(std::vector>& parallelList, - std::unordered_map>& dependencies); - hipError_t LevelOrder(std::vector& levelOrder); -}; - -class hipGraphKernelNode : public hipGraphNode { - hipKernelNodeParams* pKernelParams_; - hipFunction_t func_; - - public: - hipGraphKernelNode(const hipKernelNodeParams* pNodeParams, const hipFunction_t func) - : hipGraphNode(hipGraphNodeTypeKernel) { - pKernelParams_ = new hipKernelNodeParams(*pNodeParams); - func_ = func; - } - ~hipGraphKernelNode() { delete pKernelParams_; } - - hipError_t CreateCommand(amd::HostQueue* queue) { - commands_.reserve(1); - amd::Command* command; - hipError_t status = ihipLaunchKernelCommand( - command, func_, pKernelParams_->gridDim.x * pKernelParams_->blockDim.x, - pKernelParams_->gridDim.y * pKernelParams_->blockDim.y, - pKernelParams_->gridDim.z * pKernelParams_->blockDim.z, pKernelParams_->blockDim.x, - pKernelParams_->blockDim.y, pKernelParams_->blockDim.z, pKernelParams_->sharedMemBytes, - queue, pKernelParams_->kernelParams, pKernelParams_->extra, nullptr, nullptr, 0, 0, 0, 0, 0, - 0, 0); - commands_.emplace_back(command); - return status; - } - - void GetParams(hipKernelNodeParams* params) { - std::memcpy(params, pKernelParams_, sizeof(hipKernelNodeParams)); - } - void SetParams(hipKernelNodeParams* params) { - std::memcpy(pKernelParams_, params, sizeof(hipKernelNodeParams)); - } -}; - -class hipGraphMemcpyNode : public hipGraphNode { - hipMemcpy3DParms* pCopyParams_; - - public: - hipGraphMemcpyNode(const hipMemcpy3DParms* pCopyParams) : hipGraphNode(hipGraphNodeTypeMemcpy) { - pCopyParams_ = new hipMemcpy3DParms(*pCopyParams); - } - ~hipGraphMemcpyNode() { delete pCopyParams_; } - - hipError_t CreateCommand(amd::HostQueue* queue) { - commands_.reserve(1); - amd::Command* command; - hipError_t status = ihipMemcpy3DCommand(command, pCopyParams_, queue); - commands_.emplace_back(command); - return status; - } - - void GetParams(hipMemcpy3DParms* params) { - std::memcpy(params, pCopyParams_, sizeof(hipMemcpy3DParms)); - } - void SetParams(hipMemcpy3DParms* params) { - std::memcpy(pCopyParams_, params, sizeof(hipMemcpy3DParms)); - } -}; - - -class hipGraphMemcpyNode1D : public hipGraphNode { - void* dst_; - const void* src_; - size_t count_; - hipMemcpyKind kind_; - - public: - hipGraphMemcpyNode1D(void* dst, const void* src, size_t count, hipMemcpyKind kind) - : hipGraphNode(hipGraphNodeTypeMemcpy1D), dst_(dst), src_(src), count_(count), kind_(kind) {} - ~hipGraphMemcpyNode1D() {} - - hipError_t CreateCommand(amd::HostQueue* queue) { - commands_.reserve(1); - amd::Command* command = nullptr; - hipError_t status = ihipMemcpyCommand(command, dst_, src_, count_, kind_, *queue); - commands_.emplace_back(command); - return status; - } - - void SetParams(void* dst, const void* src, size_t count, hipMemcpyKind kind) { - dst_ = dst; - src_ = src; - count_ = count; - kind_ = kind; - } -}; - -template class hipGraphMemcpyNodeFromSymbol : public hipGraphNode { - void* dst_; - const T& symbol_; - size_t count_; - size_t offset_; - hipMemcpyKind kind_; - - public: - hipGraphMemcpyNodeFromSymbol(void* dst, const void* symbol, size_t count, size_t offset, - hipMemcpyKind kind) - : hipGraphNode(hipGraphNodeTypeMemcpyFromSymbol), - dst_(dst), - symbol_(symbol), - count_(count), - offset_(offset), - kind_(kind) {} - ~hipGraphMemcpyNodeFromSymbol() {} - - hipError_t CreateCommand(amd::HostQueue* queue); - - void SetParams(void* dst, const void* symbol, size_t count, size_t offset, hipMemcpyKind kind) { - dst_ = dst; - symbol_ = symbol; - count_ = count; - offset_ = offset; - kind_ = kind; - } -}; - -template class hipGraphMemcpyNodeToSymbol : public hipGraphNode { - const T& symbol_; - const void* src_; - size_t count_; - size_t offset_; - hipMemcpyKind kind_; - - public: - hipGraphMemcpyNodeToSymbol(const T& symbol, void* src, size_t count, size_t offset, - hipMemcpyKind kind) - : hipGraphNode(hipGraphNodeTypeMemcpyToSymbol), - symbol_(symbol), - src_(src), - count_(count), - offset_(offset), - kind_(kind) {} - ~hipGraphMemcpyNodeToSymbol() {} - - hipError_t CreateCommand(amd::HostQueue* queue); - - void SetParams(const T& symbol, void* src, size_t count, size_t offset, hipMemcpyKind kind) { - symbol_ = symbol; - src_ = src; - count_ = count; - offset_ = offset; - kind_ = kind; - } -}; - -class hipGraphMemsetNode : public hipGraphNode { - hipMemsetParams* pMemsetParams_; - - public: - hipGraphMemsetNode(const hipMemsetParams* pMemsetParams) : hipGraphNode(hipGraphNodeTypeMemset) { - pMemsetParams_ = new hipMemsetParams(*pMemsetParams); - } - ~hipGraphMemsetNode() { delete pMemsetParams_; } - - hipError_t CreateCommand(amd::HostQueue* queue) { - if (pMemsetParams_->height == 1) { - return ihipMemsetCommand(commands_, pMemsetParams_->dst, pMemsetParams_->value, - pMemsetParams_->elementSize, - pMemsetParams_->width * pMemsetParams_->elementSize, queue); - } else { - return ihipMemset3DCommand(commands_, - {pMemsetParams_->dst, pMemsetParams_->pitch, pMemsetParams_->width, - pMemsetParams_->height}, - pMemsetParams_->elementSize, - {pMemsetParams_->width, pMemsetParams_->height, 1}, queue); - } - return hipSuccess; - } - - void GetParams(hipMemsetParams* params) { - std::memcpy(params, pMemsetParams_, sizeof(hipMemsetParams)); - } - void SetParams(hipMemsetParams* params) { - std::memcpy(pMemsetParams_, params, sizeof(hipMemsetParams)); - } -}; - -class hipGraphEventRecordNode : public hipGraphNode { - hipEvent_t event_; - - public: - hipGraphEventRecordNode(hipEvent_t event) - : hipGraphNode(hipGraphNodeTypeEventRecord), event_(event) {} - ~hipGraphEventRecordNode() {} - - hipError_t CreateCommand(amd::HostQueue* queue); - - void GetParams(hipEvent_t* event) { *event = event_; } - void SetParams(hipEvent_t event) { event_ = event; } -}; - -class hipGraphEventWaitNode : public hipGraphNode { - hipEvent_t event_; - - public: - hipGraphEventWaitNode(hipEvent_t event) - : hipGraphNode(hipGraphNodeTypeWaitEvent), event_(event) {} - ~hipGraphEventWaitNode() {} - - hipError_t CreateCommand(amd::HostQueue* queue); - - void GetParams(hipEvent_t* event) { *event = event_; } - void SetParams(hipEvent_t event) { event_ = event; } -}; - -class hipGraphHostNode : public hipGraphNode { - hipHostNodeParams* pNodeParams_; - - public: - hipGraphHostNode(const hipHostNodeParams* pNodeParams) : hipGraphNode(hipGraphNodeTypeHost) { - pNodeParams_ = new hipHostNodeParams(*pNodeParams); - } - ~hipGraphHostNode() { delete pNodeParams_; } - - hipError_t CreateCommand(amd::HostQueue* queue); - - void GetParams(hipHostNodeParams* params) { - std::memcpy(params, pNodeParams_, sizeof(hipHostNodeParams)); - } - void SetParams(hipHostNodeParams* params) { - std::memcpy(pNodeParams_, params, sizeof(hipHostNodeParams)); - } -}; - -class hipGraphExec { - std::vector> parallelLists_; - std::vector levelOrder_; - std::unordered_map> nodeWaitLists_; - std::vector parallelQueues_; - static std::unordered_map activeGraphExec_; - amd::Command::EventWaitList graphLastCmdWaitList_; - amd::Command* lastEnqueuedGraphCmd_; - std::atomic bExecPending_; - amd::Command* rootCommand_; - - public: - hipGraphExec(std::vector& levelOrder, std::vector>& lists, - std::unordered_map>& nodeWaitLists) - : parallelLists_(lists), - levelOrder_(levelOrder), - nodeWaitLists_(nodeWaitLists), - lastEnqueuedGraphCmd_(nullptr), - rootCommand_(nullptr) { - bExecPending_.store(false); - } - - ~hipGraphExec() { - for (auto queue : parallelQueues_) { - queue->release(); - } - for (auto node : levelOrder_) { - delete node; - } - } - - hipError_t CreateQueues(); - hipError_t FillCommands(); - hipError_t Init(); - hipError_t UpdateGraphToWaitOnRoot(); - hipError_t Run(hipStream_t stream); - static void ResetGraph(cl_event event, cl_int command_exec_status, void* user_data); -}; diff --git a/src/hipamd/src/hip_hcc.def.in b/src/hipamd/src/hip_hcc.def.in deleted file mode 100755 index 559a08289d..0000000000 --- a/src/hipamd/src/hip_hcc.def.in +++ /dev/null @@ -1,293 +0,0 @@ -EXPORTS -hipChooseDevice -hipCtxCreate -hipCtxDestroy -hipCtxDisablePeerAccess -hipCtxEnablePeerAccess -hipCtxGetApiVersion -hipCtxGetCacheConfig -hipCtxGetCurrent -hipCtxGetDevice -hipCtxGetFlags -hipCtxGetSharedMemConfig -hipCtxPopCurrent -hipCtxPushCurrent -hipCtxSetCacheConfig -hipCtxSetCurrent -hipCtxSetSharedMemConfig -hipCtxSynchronize -hipDeviceCanAccessPeer -hipDeviceComputeCapability -hipDeviceDisablePeerAccess -hipDeviceEnablePeerAccess -hipDeviceGet -hipDeviceGetAttribute -hipDeviceGetByPCIBusId -hipDeviceGetCacheConfig -hipDeviceGetStreamPriorityRange -hipDeviceGetLimit -hipDeviceGetName -hipDeviceGetPCIBusId -hipDeviceGetSharedMemConfig -hipDeviceGetP2PAttribute -hipDevicePrimaryCtxGetState -hipDevicePrimaryCtxRelease -hipDevicePrimaryCtxReset -hipDevicePrimaryCtxRetain -hipDevicePrimaryCtxSetFlags -hipDeviceReset -hipDeviceSetCacheConfig -hipDeviceSetSharedMemConfig -hipDeviceSynchronize -hipDeviceTotalMem -hipDriverGetVersion -hipEventCreate -hipEventCreateWithFlags -hipEventDestroy -hipEventElapsedTime -hipEventQuery -hipEventRecord -hipEventSynchronize -hipExtGetLinkTypeAndHopCount -hipExtLaunchMultiKernelMultiDevice -hipExtMallocWithFlags -hipExtModuleLaunchKernel -hipExtLaunchKernel -hipFree -hipFreeArray -hipFuncSetAttribute -hipFuncSetCacheConfig -hipFuncSetSharedMemConfig -hipGetDevice -hipGetDeviceCount -hipGetDeviceProperties -hipGetErrorName -hipGetErrorString -hipGetLastError -hipMemAllocHost -hipHostAlloc -hipHostFree -hipHostGetDevicePointer -hipHostGetFlags -hipHostMalloc -hipHostRegister -hipHostUnregister -hipInit -hipIpcCloseMemHandle -hipIpcGetMemHandle -hipIpcOpenMemHandle -hipIpcGetEventHandle -hipIpcOpenEventHandle -hipMalloc -hipMalloc3D -hipMalloc3DArray -hipMallocManaged -hipArrayCreate -hipArray3DCreate -hipArrayDestroy -hipMallocArray -hipMemAdvise -hipMemAllocPitch -hipMallocPitch -hipMemcpy -hipMemcpyWithStream -hipMemcpyParam2D -hipMemcpy2D -hipMemcpy2DAsync -hipMemcpy2DToArray -hipMemcpy2DToArrayAsync -hipMemcpy3D -hipMemcpy3DAsync -hipDrvMemcpy3D -hipDrvMemcpy3DAsync -hipMemcpyAsync -hipMemcpyDtoD -hipMemcpyDtoDAsync -hipMemcpyDtoH -hipMemcpyDtoHAsync -hipMemcpyFromSymbol -hipMemcpyFromSymbolAsync -hipMemcpyHtoD -hipMemcpyHtoDAsync -hipMemcpyPeer -hipMemcpyPeerAsync -hipMemcpyToArray -hipMemcpyFromArray -hipMemcpyToSymbol -hipMemcpyToSymbolAsync -hipMemGetAddressRange -hipGetSymbolAddress -hipGetSymbolSize -hipMemGetInfo -hipMemPrefetchAsync -hipMemPtrGetInfo -hipMemRangeGetAttribute -hipMemRangeGetAttributes -hipMemset -hipMemsetAsync -hipMemsetD8 -hipMemsetD8Async -hipMemsetD16 -hipMemsetD16Async -hipMemsetD32 -hipMemsetD32Async -hipMemset2D -hipMemset2DAsync -hipMemset3D -hipMemset3DAsync -hipModuleGetFunction -hipModuleGetGlobal -hipModuleGetTexRef -hipModuleLaunchKernel -hipModuleLaunchKernelExt -hipLaunchCooperativeKernel -hipLaunchCooperativeKernelMultiDevice -hipHccModuleLaunchKernel -hipModuleLoad -hipModuleLoadData -hipModuleLoadDataEx -hipModuleUnload -hipModuleOccupancyMaxPotentialBlockSize -hipModuleOccupancyMaxPotentialBlockSizeWithFlags -hipModuleOccupancyMaxActiveBlocksPerMultiprocessor -hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags -hipOccupancyMaxPotentialBlockSize -hipOccupancyMaxActiveBlocksPerMultiprocessor -hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags -hipFuncGetAttribute -hipFuncGetAttributes -hipPeekAtLastError -hipPointerGetAttributes -hipProfilerStart -hipProfilerStop -hipRuntimeGetVersion -hipGetDeviceFlags -hipSetDevice -hipSetDeviceFlags -hipStreamAddCallback -hipStreamAttachMemAsync -hipStreamCreate -hipStreamCreateWithFlags -hipStreamCreateWithPriority -hipStreamDestroy -hipStreamGetFlags -hipStreamQuery -hipStreamSynchronize -hipStreamWaitEvent -__hipPopCallConfiguration -__hipPushCallConfiguration -__hipRegisterFatBinary -__hipRegisterFunction -__hipRegisterVar -__hipRegisterSurface -__hipRegisterTexture -__hipRegisterManagedVar -__hipUnregisterFatBinary -hipConfigureCall -hipSetupArgument -hipLaunchByPtr -hipLaunchKernel -hipRegisterApiCallback -hipRemoveApiCallback -hipRegisterActivityCallback -hipRemoveActivityCallback -hipApiName -hipKernelNameRef -hipBindTexture -hipBindTexture2D -hipBindTextureToArray -hipBindTextureToMipmappedArray -hipGetTextureAlignmentOffset -hipGetTextureReference -hipUnbindTexture -hipCreateChannelDesc -hipCreateTextureObject -hipDestroyTextureObject -hipGetChannelDesc -hipGetTextureObjectResourceDesc -hipGetTextureObjectResourceViewDesc -hipGetTextureObjectTextureDesc -hipTexRefGetAddress -hipTexRefGetAddressMode -hipTexRefGetArray -hipTexRefGetBorderColor -hipTexRefGetFilterMode -hipTexRefGetFlags -hipTexRefGetFormat -hipTexRefGetMaxAnisotropy -hipTexRefGetMipmapFilterMode -hipTexRefGetMipmapLevelBias -hipTexRefGetMipmapLevelClamp -hipTexRefGetMipmappedArray -hipTexRefSetAddress -hipTexRefSetAddress2D -hipTexRefSetAddressMode -hipTexRefSetArray -hipTexRefSetBorderColor -hipTexRefSetFilterMode -hipTexRefSetFlags -hipTexRefSetFormat -hipTexRefSetMaxAnisotropy -hipTexRefSetMipmapFilterMode -hipTexRefSetMipmapLevelBias -hipTexRefSetMipmapLevelClamp -hipTexRefSetMipmappedArray -hipProfilerStart -hipProfilerStop -hipCreateSurfaceObject -hipDestroySurfaceObject -hipInitActivityCallback -hipEnableActivityCallback -hipGetCmdName -hiprtcAddNameExpression -hiprtcCompileProgram -hiprtcCreateProgram -hiprtcDestroyProgram -hiprtcGetLoweredName -hiprtcGetProgramLog -hiprtcGetProgramLogSize -hiprtcGetCode -hiprtcGetCodeSize -hiprtcGetErrorString -hipMipmappedArrayCreate -hipMallocMipmappedArray -hipMipmappedArrayDestroy -hipFreeMipmappedArray -hipMipmappedArrayGetLevel -hipGetMipmappedArrayLevel -hipMallocHost -hipFreeHost -hipTexObjectCreate -hipTexObjectDestroy -hipTexObjectGetResourceDesc -hipTexObjectGetResourceViewDesc -hipTexObjectGetTextureDesc -hipExtStreamCreateWithCUMask -hipStreamGetPriority -hipMemcpy2DFromArray -hipMemcpy2DFromArrayAsync -hipDrvMemcpy2DUnaligned -hipMemcpyAtoH -hipMemcpyHtoA -hipMemcpyParam2DAsync -__gnu_h2f_ieee -__gnu_f2h_ieee -hipExtStreamGetCUMask -hipImportExternalMemory -hipExternalMemoryGetMappedBuffer -hipDestroyExternalMemory -hipGraphCreate -hipGraphDestroy -hipGraphAddKernelNode -hipGraphAddMemsetNode -hipGraphAddMemcpyNode -hipGraphInstantiate -hipGraphLaunch -hipStreamIsCapturing -hipStreamBeginCapture -hipStreamEndCapture -hipGraphExecDestroy -hipImportExternalSemaphore -hipSignalExternalSemaphoresAsync -hipWaitExternalSemaphoresAsync -hipDestroyExternalSemaphore diff --git a/src/hipamd/src/hip_hcc.map.in b/src/hipamd/src/hip_hcc.map.in deleted file mode 100755 index e7d761d820..0000000000 --- a/src/hipamd/src/hip_hcc.map.in +++ /dev/null @@ -1,310 +0,0 @@ -{ -global: - hipChooseDevice; - hipCtxCreate; - hipCtxDestroy; - hipCtxDisablePeerAccess; - hipCtxEnablePeerAccess; - hipCtxGetApiVersion; - hipCtxGetCacheConfig; - hipCtxGetCurrent; - hipCtxGetDevice; - hipCtxGetFlags; - hipCtxGetSharedMemConfig; - hipCtxPopCurrent; - hipCtxPushCurrent; - hipCtxSetCacheConfig; - hipCtxSetCurrent; - hipCtxSetSharedMemConfig; - hipCtxSynchronize; - hipDeviceCanAccessPeer; - hipDeviceComputeCapability; - hipDeviceDisablePeerAccess; - hipDeviceEnablePeerAccess; - hipDeviceGet; - hipDeviceGetAttribute; - hipDeviceGetByPCIBusId; - hipDeviceGetCacheConfig; - hipDeviceGetStreamPriorityRange; - hipDeviceGetLimit; - hipDeviceGetName; - hipDeviceGetPCIBusId; - hipDeviceGetSharedMemConfig; - hipDeviceGetP2PAttribute; - hipDevicePrimaryCtxGetState; - hipDevicePrimaryCtxRelease; - hipDevicePrimaryCtxReset; - hipDevicePrimaryCtxRetain; - hipDevicePrimaryCtxSetFlags; - hipDeviceReset; - hipDeviceSetCacheConfig; - hipDeviceSetSharedMemConfig; - hipDeviceSynchronize; - hipDeviceTotalMem; - hipDriverGetVersion; - hipEventCreate; - hipEventCreateWithFlags; - hipEventDestroy; - hipEventElapsedTime; - hipEventQuery; - hipEventRecord; - hipEventSynchronize; - hipExtGetLinkTypeAndHopCount; - hipExtLaunchMultiKernelMultiDevice; - hipExtMallocWithFlags; - hipExtModuleLaunchKernel; - hipExtLaunchKernel; - hipFree; - hipFreeArray; - hipFuncSetAttribute; - hipFuncSetCacheConfig; - hipFuncSetSharedMemConfig; - hipGetDevice; - hipGetDeviceCount; - hipGetDeviceProperties; - hipGetErrorName; - hipGetErrorString; - hipGetLastError; - hipMemAdvise; - hipMemAllocHost; - hipHostAlloc; - hipHostFree; - hipHostGetDevicePointer; - hipHostGetFlags; - hipHostMalloc; - hipHostRegister; - hipHostUnregister; - hipInit; - hipIpcCloseMemHandle; - hipIpcGetMemHandle; - hipIpcOpenMemHandle; - hipIpcGetEventHandle; - hipIpcOpenEventHandle; - hipMalloc; - hipMalloc3D; - hipMalloc3DArray; - hipMallocManaged; - hipArrayCreate; - hipArrayDestroy; - hipArray3DCreate; - hipMallocArray; - hipMallocPitch; - hipMemAllocPitch; - hipMemcpy; - hipMemcpyWithStream; - hipMemcpyParam2D; - hipMemcpy2D; - hipMemcpy2DAsync; - hipMemcpy2DToArray; - hipMemcpy2DToArrayAsync; - hipDrvMemcpy2DUnaligned; - hipMemcpy3D; - hipMemcpy3DAsync; - hipDrvMemcpy3D; - hipDrvMemcpy3DAsync; - hipMemcpyAsync; - hipMemcpyDtoD; - hipMemcpyDtoDAsync; - hipMemcpyDtoH; - hipMemcpyDtoHAsync; - hipMemcpyFromSymbol; - hipMemcpyFromSymbolAsync; - hipMemcpyHtoD; - hipMemcpyHtoDAsync; - hipMemcpyPeer; - hipMemcpyPeerAsync; - hipMemcpyToArray; - hipMemcpyFromArray; - hipMemcpyToSymbol; - hipMemcpyToSymbolAsync; - hipMemGetAddressRange; - hipGetSymbolAddress; - hipGetSymbolSize; - hipMemGetInfo; - hipMemPrefetchAsync; - hipMemPtrGetInfo; - hipMemRangeGetAttribute; - hipMemRangeGetAttributes; - hipMemset; - hipMemsetAsync; - hipMemsetD8; - hipMemsetD8Async; - hipMemsetD16; - hipMemsetD16Async; - hipMemsetD32; - hipMemsetD32Async; - hipMemset2D; - hipMemset2DAsync; - hipMemset3D; - hipMemset3DAsync; - hipModuleGetFunction; - hipModuleGetGlobal; - hipModuleGetTexRef; - hipModuleLaunchKernel; - hipModuleLaunchKernelExt; - hipLaunchCooperativeKernel; - hipLaunchCooperativeKernelMultiDevice; - hipModuleLoad; - hipModuleLoadData; - hipModuleLoadDataEx; - hipModuleUnload; - hipModuleOccupancyMaxPotentialBlockSize; - hipModuleOccupancyMaxPotentialBlockSizeWithFlags; - hipModuleOccupancyMaxActiveBlocksPerMultiprocessor; - hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags; - hipOccupancyMaxPotentialBlockSize; - hipOccupancyMaxActiveBlocksPerMultiprocessor; - hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags; - hipFuncGetAttribute; - hipFuncGetAttributes; - hipPeekAtLastError; - hipPointerGetAttributes; - hipProfilerStart; - hipProfilerStop; - hipRuntimeGetVersion; - hipGetDeviceFlags; - hipSetDevice; - hipSetDeviceFlags; - hipStreamAddCallback; - hipStreamAttachMemAsync; - hipStreamCreate; - hipStreamCreateWithFlags; - hipStreamCreateWithPriority; - hipStreamDestroy; - hipStreamGetFlags; - hipStreamQuery; - hipStreamSynchronize; - hipStreamWaitEvent; - hipStreamWaitValue32; - hipStreamWaitValue64; - hipStreamWriteValue32; - hipStreamWriteValue64; - __hipPopCallConfiguration; - __hipPushCallConfiguration; - __hipRegisterFatBinary; - __hipRegisterFunction; - __hipRegisterVar; - __hipRegisterSurface; - __hipRegisterTexture; - __hipRegisterManagedVar; - __hipUnregisterFatBinary; - __gnu_h2f_ieee; - __gnu_f2h_ieee; - hipConfigureCall; - hipSetupArgument; - hipLaunchByPtr; - hipLaunchKernel; - hipRegisterApiCallback; - hipRemoveApiCallback; - hipRegisterActivityCallback; - hipRemoveActivityCallback; - hipApiName; - hipKernelNameRef; - hipKernelNameRefByPtr; - hipGetStreamDeviceId; - hipProfilerStart; - hipProfilerStop; - hiprtcCompileProgram; - hiprtcCreateProgram; - hiprtcDestroyProgram; - hiprtcGetLoweredName; - hiprtcGetProgramLog; - hiprtcGetProgramLogSize; - hiprtcGetCode; - hiprtcGetCodeSize; - hiprtcGetErrorString; - hiprtcAddNameExpression; - hiprtcVersion; - hipBindTexture; - hipBindTexture2D; - hipBindTextureToArray; - hipBindTextureToMipmappedArray; - hipGetTextureAlignmentOffset; - hipGetTextureReference; - hipUnbindTexture; - hipCreateChannelDesc; - hipCreateTextureObject; - hipDestroyTextureObject; - hipGetChannelDesc; - hipGetTextureObjectResourceDesc; - hipGetTextureObjectResourceViewDesc; - hipGetTextureObjectTextureDesc; - hipTexRefGetAddress; - hipTexRefGetAddressMode; - hipTexRefGetArray; - hipTexRefGetBorderColor; - hipTexRefGetFilterMode; - hipTexRefGetFlags; - hipTexRefGetFormat; - hipTexRefGetMaxAnisotropy; - hipTexRefGetMipmapFilterMode; - hipTexRefGetMipmapLevelBias; - hipTexRefGetMipmapLevelClamp; - hipTexRefGetMipmappedArray; - hipTexRefSetAddress; - hipTexRefSetAddress2D; - hipTexRefSetAddressMode; - hipTexRefSetArray; - hipTexRefSetBorderColor; - hipTexRefSetFilterMode; - hipTexRefSetFlags; - hipTexRefSetFormat; - hipTexRefSetMaxAnisotropy; - hipTexRefSetMipmapFilterMode; - hipTexRefSetMipmapLevelBias; - hipTexRefSetMipmapLevelClamp; - hipTexRefSetMipmappedArray; - hipMipmappedArrayCreate; - hipMallocMipmappedArray; - hipMipmappedArrayDestroy; - hipFreeMipmappedArray; - hipMipmappedArrayGetLevel; - hipGetMipmappedArrayLevel; - hipMallocHost; - hipFreeHost; - hipTexObjectCreate; - hipTexObjectDestroy; - hipTexObjectGetResourceDesc; - hipTexObjectGetResourceViewDesc; - hipTexObjectGetTextureDesc; - hipImportExternalMemory; - hipExternalMemoryGetMappedBuffer; - hipDestroyExternalMemory; - hipGraphCreate; - hipGraphDestroy; - hipGraphAddKernelNode; - hipGraphAddMemsetNode; - hipGraphAddMemcpyNode; - hipGraphInstantiate; - hipGraphLaunch; - hipStreamIsCapturing; - hipStreamBeginCapture; - hipStreamEndCapture; - hipGraphExecDestroy; - hipImportExternalSemaphore; - hipSignalExternalSemaphoresAsync; - hipWaitExternalSemaphoresAsync; - hipDestroyExternalSemaphore; - extern "C++" { - hip_impl::hipLaunchKernelGGLImpl*; - hip_impl::demangle*; - hipCreateSurfaceObject*; - hipDestroySurfaceObject*; - hipHccModuleLaunchKernel*; - hipExtModuleLaunchKernel*; - hipInitActivityCallback*; - hipEnableActivityCallback*; - hipGetCmdName*; - hipExtStreamCreateWithCUMask; - hipStreamGetPriority; - hipMemcpy2DFromArray; - hipMemcpy2DFromArrayAsync; - hipMemcpyAtoH; - hipMemcpyHtoA; - hipMemcpyParam2DAsync; - __hipGetPCH; - hipExtStreamGetCUMask; - }; -local: - *; -}; diff --git a/src/hipamd/src/hip_hcc.rc b/src/hipamd/src/hip_hcc.rc deleted file mode 100644 index 009dc30c18..0000000000 --- a/src/hipamd/src/hip_hcc.rc +++ /dev/null @@ -1,75 +0,0 @@ -#define STR(__macro__) #__macro__ -#define XSTR(__macro__) STR(__macro__) - -#if defined(_DEBUG) -#define DEBUG_ONLY(x) x -#else -#define DEBUG_ONLY(x) -#endif - -#define VERSION_PREFIX_MAJOR 2 -#define VERSION_PREFIX_MINOR 0 - - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "winresrc.h" -#include "utils/versions.hpp" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 10,0,AMD_PLATFORM_BUILD_NUMBER,AMD_PLATFORM_REVISION_NUMBER - PRODUCTVERSION 10,0,AMD_PLATFORM_BUILD_NUMBER,AMD_PLATFORM_REVISION_NUMBER - FILEFLAGSMASK 0x3fL -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x40004L - FILETYPE 0x2L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "Comments", " \0" - VALUE "CompanyName", "Advanced Micro Devices Inc.\0" - VALUE "FileDescription", AMD_PLATFORM_NAME " OpenCL " XSTR(VERSION_PREFIX_MAJOR) "." XSTR(VERSION_PREFIX_MINOR) " Runtime\0" - VALUE "FileVersion", "10.0." XSTR(AMD_PLATFORM_BUILD_NUMBER) "." XSTR(AMD_PLATFORM_REVISION_NUMBER) - VALUE "InternalName", "OpenCL" - VALUE "LegalCopyright", "Copyright (C) 2011 Advanced Micro Devices Inc.\0" - VALUE "OriginalFilename", "OpenCL.dll" - VALUE "ProductName", "OpenCL " XSTR(VERSION_PREFIX_MAJOR) "." XSTR(VERSION_PREFIX_MINOR) " " AMD_PLATFORM_INFO "\0" - VALUE "ProductVersion", "10.0." XSTR(AMD_PLATFORM_BUILD_NUMBER) "." XSTR(AMD_PLATFORM_REVISION_NUMBER) - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// diff --git a/src/hipamd/src/hip_hmm.cpp b/src/hipamd/src/hip_hmm.cpp deleted file mode 100644 index 531b1efc19..0000000000 --- a/src/hipamd/src/hip_hmm.cpp +++ /dev/null @@ -1,224 +0,0 @@ -/* Copyright (c) 2020-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include "hip_internal.hpp" -#include "hip_conversions.hpp" -#include "platform/context.hpp" -#include "platform/command.hpp" -#include "platform/memory.hpp" - -// Forward declaraiton of a function -hipError_t ihipMallocManaged(void** ptr, size_t size, unsigned int align = 0); - -// Make sure HIP defines match ROCclr to avoid double conversion -static_assert(hipCpuDeviceId == amd::CpuDeviceId, "CPU device ID mismatch with ROCclr!"); -static_assert(hipInvalidDeviceId == amd::InvalidDeviceId, - "Invalid device ID mismatch with ROCclr!"); - -static_assert(static_cast(hipMemAdviseSetReadMostly) == - amd::MemoryAdvice::SetReadMostly, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemAdviseUnsetReadMostly) == - amd::MemoryAdvice::UnsetReadMostly, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemAdviseSetPreferredLocation) == - amd::MemoryAdvice::SetPreferredLocation, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemAdviseUnsetPreferredLocation) == - amd::MemoryAdvice::UnsetPreferredLocation, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemAdviseSetAccessedBy) == - amd::MemoryAdvice::SetAccessedBy, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemAdviseUnsetAccessedBy) == - amd::MemoryAdvice::UnsetAccessedBy, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemAdviseSetCoarseGrain) == - amd::MemoryAdvice::SetCoarseGrain, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemAdviseUnsetCoarseGrain) == - amd::MemoryAdvice::UnsetCoarseGrain, "Enum mismatch with ROCclr!"); - -static_assert(static_cast(hipMemRangeAttributeReadMostly) == - amd::MemRangeAttribute::ReadMostly, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemRangeAttributePreferredLocation) == - amd::MemRangeAttribute::PreferredLocation, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemRangeAttributeAccessedBy) == - amd::MemRangeAttribute::AccessedBy, "Enum mismatch with ROCclr!"); -static_assert(static_cast(hipMemRangeAttributeLastPrefetchLocation) == - amd::MemRangeAttribute::LastPrefetchLocation, "Enum mismatch with ROCclr!"); - -// ================================================================================================ -hipError_t hipMallocManaged(void** dev_ptr, size_t size, unsigned int flags) { - HIP_INIT_API(hipMallocManaged, dev_ptr, size, flags); - - if ((dev_ptr == nullptr) || (size == 0) || - ((flags != hipMemAttachGlobal) && (flags != hipMemAttachHost))) { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(ihipMallocManaged(dev_ptr, size), *dev_ptr); -} - -// ================================================================================================ -hipError_t hipMemPrefetchAsync(const void* dev_ptr, size_t count, int device, - hipStream_t stream) { - HIP_INIT_API(hipMemPrefetchAsync, dev_ptr, count, device, stream); - - if ((dev_ptr == nullptr) || (count == 0)) { - HIP_RETURN(hipErrorInvalidValue); - } - amd::HostQueue* queue = nullptr; - bool cpu_access = (device == hipCpuDeviceId) ? true : false; - - // Pick the specified stream or Null one from the provided device - if (stream != nullptr) { - queue = hip::getQueue(stream); - } else { - if (!cpu_access) { - queue = g_devices[device]->NullStream(); - } else { - queue = hip::getCurrentDevice()->NullStream(); - } - } - if (queue == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - amd::Command::EventWaitList waitList; - amd::SvmPrefetchAsyncCommand* command = - new amd::SvmPrefetchAsyncCommand(*queue, waitList, dev_ptr, count, cpu_access); - if (command == nullptr) { - return hipErrorOutOfMemory; - } - - if (!command->validateMemory()) { - delete command; - HIP_RETURN(hipErrorInvalidValue); - } - command->enqueue(); - command->release(); - - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipMemAdvise(const void* dev_ptr, size_t count, hipMemoryAdvise advice, int device) { - HIP_INIT_API(hipMemAdvise, dev_ptr, count, advice, device); - - if ((dev_ptr == nullptr) || (count == 0) || - ((device != hipCpuDeviceId) && (static_cast(device) >= g_devices.size()))) { - HIP_RETURN(hipErrorInvalidValue); - } - amd::Device* dev = (device == hipCpuDeviceId) ? - g_devices[0]->devices()[0] : g_devices[device]->devices()[0]; - bool use_cpu = (device == hipCpuDeviceId) ? true : false; - - // Set the allocation attributes in AMD HMM - if (!dev->SetSvmAttributes(dev_ptr, count, static_cast(advice), use_cpu)) { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipMemRangeGetAttribute(void* data, size_t data_size, hipMemRangeAttribute attribute, - const void* dev_ptr, size_t count) { - HIP_INIT_API(hipMemRangeGetAttribute, data, data_size, attribute, dev_ptr, count); - - if ((data == nullptr) || (data_size == 0) || (dev_ptr == nullptr) || (count == 0)) { - HIP_RETURN(hipErrorInvalidValue); - } - - // Shouldn't matter for which device the interface is called - amd::Device* dev = g_devices[0]->devices()[0]; - - // Get the allocation attribute from AMD HMM - if (!dev->GetSvmAttributes(&data, &data_size, reinterpret_cast(&attribute), 1, - dev_ptr, count)) { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipMemRangeGetAttributes(void** data, size_t* data_sizes, - hipMemRangeAttribute* attributes, size_t num_attributes, - const void* dev_ptr, size_t count) { - HIP_INIT_API(hipMemRangeGetAttributes, data, data_sizes, - attributes, num_attributes, dev_ptr, count); - - if ((data == nullptr) || (data_sizes == nullptr) || (attributes == nullptr) || - (num_attributes == 0) || (dev_ptr == nullptr) || (count == 0)) { - HIP_RETURN(hipErrorInvalidValue); - } - - // Shouldn't matter for which device the interface is called - amd::Device* dev = g_devices[0]->devices()[0]; - // Get the allocation attributes from AMD HMM - if (!dev->GetSvmAttributes(data, data_sizes, reinterpret_cast(attributes), - num_attributes, dev_ptr, count)) { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipStreamAttachMemAsync(hipStream_t stream, hipDeviceptr_t* dev_ptr, - size_t length, unsigned int flags) { - HIP_INIT_API(hipStreamAttachMemAsync, stream, dev_ptr, length, flags); - - if ((stream == nullptr) || (dev_ptr == nullptr) || (length == 0)) { - HIP_RETURN(hipErrorInvalidValue); - } - - // Unclear what should be done for this interface in AMD HMM, since it's generic SVM alloc - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t ihipMallocManaged(void** ptr, size_t size, unsigned int align) { - if (ptr == nullptr) { - return hipErrorInvalidValue; - } else if (size == 0) { - *ptr = nullptr; - return hipSuccess; - } - - assert((hip::host_device->asContext()!= nullptr) && "Current host context must be valid"); - amd::Context& ctx = *hip::host_device->asContext(); - - const amd::Device& dev = *ctx.devices()[0]; - - // For now limit to the max allocation size on the device. - // The apps should be able to go over the limit in the future - if (dev.info().maxMemAllocSize_ < size) { - return hipErrorMemoryAllocation; - } - - // Allocate SVM fine grain buffer with the forced host pointer, avoiding explicit memory - // allocation in the device driver - *ptr = amd::SvmBuffer::malloc(ctx, CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_ALLOC_HOST_PTR, - size, (align == 0) ? dev.info().memBaseAddrAlign_ : align); - if (*ptr == nullptr) { - return hipErrorMemoryAllocation; - } - - ClPrint(amd::LOG_INFO, amd::LOG_API, "%-5d: [%zx] ihipMallocManaged ptr=0x%zx", getpid(), - std::this_thread::get_id(), *ptr); - return hipSuccess; -} diff --git a/src/hipamd/src/hip_intercept.cpp b/src/hipamd/src/hip_intercept.cpp deleted file mode 100755 index 6ff64a4bc8..0000000000 --- a/src/hipamd/src/hip_intercept.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* Copyright (c) 2019-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "hip/hip_runtime.h" -#include "hip_internal.hpp" -#include "hip_platform.hpp" -#include "hip_prof_api.h" - -// HIP API callback/activity - -api_callbacks_table_t callbacks_table; - -extern const std::string& FunctionName(const hipFunction_t f); - -const char* hipKernelNameRef(const hipFunction_t f) { return FunctionName(f).c_str(); } - -int hipGetStreamDeviceId(hipStream_t stream) { - hip::Stream* s = reinterpret_cast(stream); - return (s != nullptr)? s->DeviceId() : ihipGetDevice(); -} - -const char* hipKernelNameRefByPtr(const void* hostFunction, hipStream_t stream) { - if (hostFunction == NULL) { - return NULL; - } - int deviceId = hipGetStreamDeviceId(stream); - if (deviceId == -1) { - LogPrintfError("Wrong Device Id: %d \n", deviceId); - return NULL; - } - hipFunction_t func = nullptr; - hipError_t hip_error = PlatformState::instance().getStatFunc(&func, hostFunction, deviceId); - if (hip_error != hipSuccess) { - return NULL; - } - return hipKernelNameRef(func); -} - -hipError_t hipRegisterApiCallback(uint32_t id, void* fun, void* arg) { - return callbacks_table.set_callback(id, reinterpret_cast(fun), arg) ? - hipSuccess : hipErrorInvalidValue; -} - -hipError_t hipRemoveApiCallback(uint32_t id) { - return callbacks_table.set_callback(id, NULL, NULL) ? hipSuccess : hipErrorInvalidValue; -} - -hipError_t hipRegisterActivityCallback(uint32_t id, void* fun, void* arg) { - return callbacks_table.set_activity(id, reinterpret_cast(fun), arg) ? - hipSuccess : hipErrorInvalidValue; -} - -hipError_t hipRemoveActivityCallback(uint32_t id) { - return callbacks_table.set_activity(id, NULL, NULL) ? hipSuccess : hipErrorInvalidValue; -} - -hipError_t hipEnableTracing(bool enabled) { - callbacks_table.set_enabled(enabled); - return hipSuccess; -} - -const char* hipApiName(uint32_t id) { - return hip_api_name(id); -} diff --git a/src/hipamd/src/hip_internal.hpp b/src/hipamd/src/hip_internal.hpp deleted file mode 100755 index 577039ded2..0000000000 --- a/src/hipamd/src/hip_internal.hpp +++ /dev/null @@ -1,353 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef HIP_SRC_HIP_INTERNAL_H -#define HIP_SRC_HIP_INTERNAL_H - -#include "vdi_common.hpp" -#include "hip_prof_api.h" -#include "trace_helper.h" -#include "utils/debug.hpp" -#include "hip_formatting.hpp" - -#include "hip_graph_capture.hpp" - -#include -#include -#include -#include -#include -#ifdef _WIN32 -#include -#else -#include -#endif - -#define KNRM "\x1B[0m" -#define KRED "\x1B[31m" -#define KGRN "\x1B[32m" -#define KYEL "\x1B[33m" -#define KBLU "\x1B[34m" -#define KMAG "\x1B[35m" -#define KCYN "\x1B[36m" -#define KWHT "\x1B[37m" - -/*! IHIP IPC MEMORY Structure */ -#define IHIP_IPC_MEM_HANDLE_SIZE 32 -#define IHIP_IPC_MEM_RESERVED_SIZE LP64_SWITCH(24,16) - -typedef struct ihipIpcMemHandle_st { - char ipc_handle[IHIP_IPC_MEM_HANDLE_SIZE]; ///< ipc memory handle on ROCr - size_t psize; - size_t poffset; - char reserved[IHIP_IPC_MEM_RESERVED_SIZE]; -} ihipIpcMemHandle_t; - -#define IHIP_IPC_EVENT_HANDLE_SIZE 32 -#define IHIP_IPC_EVENT_RESERVED_SIZE LP64_SWITCH(28,24) -typedef struct ihipIpcEventHandle_st { - //hsa_amd_ipc_signal_t ipc_handle; ///< ipc signal handle on ROCr - //char ipc_handle[IHIP_IPC_EVENT_HANDLE_SIZE]; - //char reserved[IHIP_IPC_EVENT_RESERVED_SIZE]; - char shmem_name[IHIP_IPC_EVENT_HANDLE_SIZE]; -}ihipIpcEventHandle_t; - -#ifdef _WIN32 - inline int getpid() { return _getpid(); } -#endif - -#define HIP_INIT() \ - std::call_once(hip::g_ihipInitialized, hip::init); \ - if (hip::g_device == nullptr && g_devices.size() > 0) { \ - hip::g_device = g_devices[0]; \ - } - -#define HIP_API_PRINT(...) \ - uint64_t startTimeUs=0 ; HIPPrintDuration(amd::LOG_INFO, amd::LOG_API, &startTimeUs, "%-5d: [%zx] %s%s ( %s )%s", getpid(), std::this_thread::get_id(), KGRN, \ - __func__, ToString( __VA_ARGS__ ).c_str(),KNRM); - -#define HIP_ERROR_PRINT(err, ...) \ - ClPrint(amd::LOG_INFO, amd::LOG_API, "%-5d: [%zx] %s: Returned %s : %s", getpid(), std::this_thread::get_id(), \ - __func__, hipGetErrorName(err), ToString( __VA_ARGS__ ).c_str()); - -// This macro should be called at the beginning of every HIP API. -#define HIP_INIT_API(cid, ...) \ - HIP_API_PRINT(__VA_ARGS__) \ - amd::Thread* thread = amd::Thread::current(); \ - if (!VDI_CHECK_THREAD(thread)) { \ - HIP_RETURN(hipErrorOutOfMemory); \ - } \ - HIP_INIT() \ - HIP_CB_SPAWNER_OBJECT(cid); - -#define HIP_RETURN_DURATION(ret, ...) \ - hip::g_lastError = ret; \ - HIPPrintDuration(amd::LOG_INFO, amd::LOG_API, &startTimeUs, "%-5d: [%zx] %s: Returned %s : %s", getpid(), std::this_thread::get_id(), \ - __func__, hipGetErrorName(hip::g_lastError), ToString( __VA_ARGS__ ).c_str()); \ - return hip::g_lastError; - -#define HIP_RETURN(ret, ...) \ - hip::g_lastError = ret; \ - HIP_ERROR_PRINT(hip::g_lastError, __VA_ARGS__) \ - return hip::g_lastError; - -#define HIP_RETURN_ONFAIL(func) \ - do { \ - hipError_t herror = (func); \ - if (herror != hipSuccess) { \ - HIP_RETURN(herror); \ - } \ - } while (0); - -// Cannot be use in place of HIP_RETURN. -// Refrain from using for external HIP APIs -#define IHIP_RETURN_ONFAIL(func) \ - do { \ - hipError_t herror = (func); \ - if (herror != hipSuccess) { \ - return herror; \ - } \ - } while (0); - -#define STREAM_CAPTURE(name, stream, ...) \ - if (stream != nullptr && \ - reinterpret_cast(stream)->GetCaptureStatus() == \ - hipStreamCaptureStatusActive) { \ - hipError_t status = capture##name(stream, ##__VA_ARGS__); \ - HIP_RETURN(status); \ - } - -#define EVENT_CAPTURE(name, event, ...) \ - if (event != nullptr && reinterpret_cast(event)->GetCaptureStatus() == true) { \ - hipError_t status = capture##name(event, ##__VA_ARGS__); \ - HIP_RETURN(status); \ - } - -namespace hc { -class accelerator; -class accelerator_view; -}; - -namespace hip { - class Device; - - class Stream { - public: - enum Priority : int { High = -1, Normal = 0, Low = 1 }; - - private: - amd::HostQueue* queue_; - mutable amd::Monitor lock_; - Device* device_; - Priority priority_; - unsigned int flags_; - bool null_; - const std::vector cuMask_; - - /// Stream capture related parameters - - /// Current capture status of the stream - hipStreamCaptureStatus captureStatus_; - /// Graph that is constructed with capture - hipGraph_t pCaptureGraph_; - /// Based on mode stream capture places restrictions on API calls that can be made within or - /// concurrently - hipStreamCaptureMode captureMode_; - bool originStream_; - /// Origin sream has no parent. Parent stream for the derived captured streams with event - /// dependencies - hipStream_t parentStream_; - /// Last graph node captured in the stream - std::vector lastCapturedNodes_; - /// Derived streams/Paralell branches from the origin stream - std::vector parallelCaptureStreams_; - /// Capture events - std::vector captureEvents_; - - public: - Stream(Device* dev, Priority p = Priority::Normal, unsigned int f = 0, bool null_stream = false, - const std::vector& cuMask = {}, - hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone); - ~Stream(); - /// Creates the hip stream object, including AMD host queue - bool Create(); - - /// Get device AMD host queue object. The method can allocate the queue - amd::HostQueue* asHostQueue(bool skip_alloc = false); - - void Finish() const; - /// Get device ID associated with the current stream; - int DeviceId() const; - /// Get device ID associated with a stream; - static int DeviceId(const hipStream_t hStream); - /// Returns if stream is null stream - bool Null() const { return null_; } - /// Returns the lock object for the current stream - amd::Monitor& Lock() const { return lock_; } - /// Returns the creation flags for the current stream - unsigned int Flags() const { return flags_; } - /// Returns the priority for the current stream - Priority GetPriority() const { return priority_; } - /// Returns the CU mask for the current stream - const std::vector GetCUMask() const { return cuMask_; } - - /// Sync all non-blocking streams - static void syncNonBlockingStreams(); - - /// Returns capture status of the current stream - hipStreamCaptureStatus GetCaptureStatus() const { return captureStatus_; } - /// Returns capture mode of the current stream - hipStreamCaptureMode GetCaptureMode() const { return captureMode_; } - /// Returns if stream is origin stream - bool IsOriginStream() const { return originStream_; } - void SetOriginStream() { originStream_ = true; } - /// Returns captured graph - hipGraph_t GetCaptureGraph() const { return pCaptureGraph_; } - /// Returns last captured graph node - std::vector GetLastCapturedNodes() const { return lastCapturedNodes_; } - /// Set last captured graph node - void SetLastCapturedNode(hipGraphNode_t graphNode) { - lastCapturedNodes_.clear(); - lastCapturedNodes_.push_back(graphNode); - } - /// Append captured node via the wait event cross stream - void AddCrossCapturedNode(std::vector graphNodes) { - for (auto node : graphNodes) { - lastCapturedNodes_.push_back(node); - } - } - /// Set graph that is being captured - void SetCaptureGraph(hipGraph_t pGraph) { - pCaptureGraph_ = pGraph; - captureStatus_ = hipStreamCaptureStatusActive; - } - /// reset capture parameters - hipError_t EndCapture(); - /// Set capture status - void SetCaptureStatus(hipStreamCaptureStatus captureStatus) { captureStatus_ = captureStatus; } - /// Set capture mode - void SetCaptureMode(hipStreamCaptureMode captureMode) { captureMode_ = captureMode; } - /// Set parent stream - void SetParentStream(hipStream_t parentStream) { parentStream_ = parentStream; } - /// Get parent stream - hipStream_t GetParentStream() { return parentStream_; } - }; - - /// HIP Device class - class Device { - amd::Monitor lock_{"Device lock"}; - /// ROCclr context - amd::Context* context_; - /// Device's ID - /// Store it here so we don't have to loop through the device list every time - int deviceId_; - /// ROCclr host queue for default streams - Stream null_stream_; - /// Store device flags - unsigned int flags_; - /// Maintain list of user enabled peers - std::list userEnabledPeers; - - public: - Device(amd::Context* ctx, int devId): - context_(ctx), deviceId_(devId), null_stream_(this, Stream::Priority::Normal, 0, true), flags_(hipDeviceScheduleSpin) - { assert(ctx != nullptr); } - ~Device() {} - - amd::Context* asContext() const { return context_; } - int deviceId() const { return deviceId_; } - void retain() const { context_->retain(); } - void release() const { context_->release(); } - const std::vector& devices() const { return context_->devices(); } - hipError_t EnablePeerAccess(int peerDeviceId){ - amd::ScopedLock lock(lock_); - bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end()); - if (found) { - return hipErrorPeerAccessAlreadyEnabled; - } - userEnabledPeers.push_back(peerDeviceId); - return hipSuccess; - } - hipError_t DisablePeerAccess(int peerDeviceId) { - amd::ScopedLock lock(lock_); - bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end()); - if (found) { - userEnabledPeers.remove(peerDeviceId); - return hipSuccess; - } else { - return hipErrorPeerAccessNotEnabled; - } - } - unsigned int getFlags() const { return flags_; } - void setFlags(unsigned int flags) { flags_ = flags; } - amd::HostQueue* NullStream(bool skip_alloc = false); - }; - - extern std::once_flag g_ihipInitialized; - /// Current thread's device - extern thread_local Device* g_device; - extern thread_local hipError_t g_lastError; - /// Device representing the host - for pinned memory - extern Device* host_device; - - extern void init(); - - extern Device* getCurrentDevice(); - - extern void setCurrentDevice(unsigned int index); - - /// Get ROCclr queue associated with hipStream - /// Note: This follows the CUDA spec to sync with default streams - /// and Blocking streams - extern amd::HostQueue* getQueue(hipStream_t s); - /// Get default stream associated with the ROCclr context - extern amd::HostQueue* getNullStream(amd::Context&); - /// Get default stream of the thread - extern amd::HostQueue* getNullStream(); - /// Check if stream is valid - extern bool isValid(hipStream_t stream); -}; - -struct ihipExec_t { - dim3 gridDim_; - dim3 blockDim_; - size_t sharedMem_; - hipStream_t hStream_; - std::vector arguments_; -}; - -/// Wait all active streams on the blocking queue. The method enqueues a wait command and -/// doesn't stall the current thread -extern void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null_stream = false); - -extern std::vector g_devices; -extern hipError_t ihipDeviceGetCount(int* count); -extern int ihipGetDevice(); - -extern hipError_t ihipMalloc(void** ptr, size_t sizeBytes, unsigned int flags); -extern amd::Memory* getMemoryObject(const void* ptr, size_t& offset); -extern amd::Memory* getMemoryObjectWithOffset(const void* ptr, const size_t size); - -constexpr bool kOptionChangeable = true; -constexpr bool kNewDevProg = false; - -constexpr bool kMarkerDisableFlush = true; //!< Avoids command batch flush in ROCclr - -#endif // HIP_SRC_HIP_INTERNAL_H diff --git a/src/hipamd/src/hip_memory.cpp b/src/hipamd/src/hip_memory.cpp deleted file mode 100755 index 0d49cb942c..0000000000 --- a/src/hipamd/src/hip_memory.cpp +++ /dev/null @@ -1,2852 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include "hip_internal.hpp" -#include "hip_platform.hpp" -#include "hip_conversions.hpp" -#include "platform/context.hpp" -#include "platform/command.hpp" -#include "platform/memory.hpp" -#include "amdocl/cl_vk_amd.hpp" - -// ================================================================================================ -amd::Memory* getMemoryObject(const void* ptr, size_t& offset) { - amd::Memory *memObj = amd::MemObjMap::FindMemObj(ptr); - if (memObj != nullptr) { - const char* hostPtr = reinterpret_cast(ptr); - const char* hostMem = reinterpret_cast(memObj->getHostMem()); - //Prepinned memory - if ((hostMem != nullptr) && - (hostPtr >= hostMem && hostPtr <= (hostMem + memObj->getSize()))) { - offset = reinterpret_cast(hostPtr) - reinterpret_cast(hostMem); - } - else { - //SVM ptr or device ptr mapped from host - const void *devPtr = reinterpret_cast - (memObj->getDeviceMemory(*memObj->getContext().devices()[0])->virtualAddress()); - if (devPtr != nullptr) { - offset = reinterpret_cast(ptr) - reinterpret_cast(devPtr); - } - else { - ShouldNotReachHere(); - } - } - } else { - // If memObj not found, use arena_mem_obj. arena_mem_obj is null, if HMM and Xnack is disabled. - memObj = (hip::getCurrentDevice()->asContext()->svmDevices()[0])->GetArenaMemObj(ptr, offset); - } - return memObj; -} - -// ================================================================================================ -amd::Memory* getMemoryObjectWithOffset(const void* ptr, const size_t size) { - size_t offset; - amd::Memory* memObj = getMemoryObject(ptr, offset); - - if (memObj != nullptr) { - assert(size <= (memObj->getSize() - offset)); - memObj = new (memObj->getContext()) amd::Buffer(*memObj, memObj->getMemFlags(), offset, size); - if (memObj == nullptr) {; - return nullptr; - } - - if (!memObj->create(nullptr)) { - memObj->release(); - return nullptr; - } - } - - return memObj; -} - -// ================================================================================================ -hipError_t ihipFree(void *ptr) -{ - if (ptr == nullptr) { - return hipSuccess; - } - - size_t offset = 0; - amd::Memory* memory_object = getMemoryObject(ptr, offset); - - if (memory_object != nullptr) { - // Check if it's an allocation in system memory and can be shared across all devices - if (memory_object->getMemFlags() & CL_MEM_SVM_FINE_GRAIN_BUFFER) { - for (auto& dev : g_devices) { - // Skip stream allocation, since if it wasn't allocated until free, then the device - // wasn't used - constexpr bool SkipStreamAlloc = true; - amd::HostQueue* queue = dev->NullStream(SkipStreamAlloc); - if (queue != nullptr) { - queue->finish(); - } - } - } else { - // Wait on the device, associated with the current memory object - hip::getNullStream(memory_object->getContext())->finish(); - } - amd::SvmBuffer::free(memory_object->getContext(), ptr); - return hipSuccess; - } - return hipErrorInvalidValue; -} - -hipError_t hipImportExternalMemory(hipExternalMemory_t* extMem_out, const hipExternalMemoryHandleDesc* memHandleDesc) { - HIP_INIT_API(hipImportExternalMemory, extMem_out, memHandleDesc); - - size_t sizeBytes = memHandleDesc->size; - amd::Context& amdContext = *hip::getCurrentDevice()->asContext(); - - amd::BufferVk* pBufferVk = nullptr; -#ifdef _WIN32 - pBufferVk = new (amdContext) amd::BufferVk(amdContext, sizeBytes, memHandleDesc->handle.win32.handle); -#else - pBufferVk = new (amdContext) amd::BufferVk(amdContext, sizeBytes, memHandleDesc->handle.fd); -#endif - - if (!pBufferVk) { - HIP_RETURN(hipErrorOutOfMemory); - } - - if (!pBufferVk->create()) { - pBufferVk->release(); - HIP_RETURN(hipErrorOutOfMemory); - } - *extMem_out = pBufferVk; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipExternalMemoryGetMappedBuffer(void **devPtr, hipExternalMemory_t extMem, const hipExternalMemoryBufferDesc *bufferDesc) { - HIP_INIT_API(hipExternalMemoryGetMappedBuffer, devPtr, extMem, bufferDesc); - - if (extMem == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - amd::BufferVk *buf = reinterpret_cast(extMem); - const device::Memory* devMem = buf->getDeviceMemory(*hip::getCurrentDevice()->devices()[0]); - if (devMem != nullptr) { - *devPtr = reinterpret_cast(devMem->virtualAddress()); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDestroyExternalMemory(hipExternalMemory_t extMem) { - HIP_INIT_API(hipDestroyExternalMemory, extMem); - - if (extMem == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - reinterpret_cast(extMem)->release(); - - HIP_RETURN(hipSuccess); -} - - -hipError_t hipImportExternalSemaphore(hipExternalSemaphore_t* extSem_out, - const hipExternalSemaphoreHandleDesc* semHandleDesc) -{ - HIP_INIT_API(hipImportExternalSemaphore, extSem_out, semHandleDesc); - if (extSem_out == nullptr || semHandleDesc == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - amd::Device* device = hip::getCurrentDevice()->devices()[0]; - -#ifdef _WIN32 - if (device->importExtSemaphore(extSem_out, semHandleDesc->handle.win32.handle)) { -#else - if (device->importExtSemaphore( - extSem_out, semHandleDesc->handle.fd)) { -#endif - HIP_RETURN(hipSuccess); - } - HIP_RETURN(hipErrorInvalidValue); -} - - -hipError_t hipSignalExternalSemaphoresAsync( - const hipExternalSemaphore_t* extSemArray, const hipExternalSemaphoreSignalParams* paramsArray, - unsigned int numExtSems, hipStream_t stream ) -{ - HIP_INIT_API(hipSignalExternalSemaphoresAsync, extSemArray, paramsArray, numExtSems, stream); - if (extSemArray == nullptr || paramsArray == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - amd::HostQueue* queue = hip::getQueue(stream); - const amd::Device& device = queue->vdev()->device(); - - for (unsigned int i = 0; i < numExtSems; i++) { - if (extSemArray[i] != nullptr) { - amd::ExternalSemaphoreCmd* command = - new amd::ExternalSemaphoreCmd(*queue, extSemArray[i], paramsArray[i].params.fence.value, - amd::ExternalSemaphoreCmd::COMMAND_SIGNAL_EXTSEMAPHORE); - if (command == nullptr) { - return hipErrorOutOfMemory; - } - command->enqueue(); - command->release(); - } else { - HIP_RETURN(hipErrorInvalidValue); - } - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipWaitExternalSemaphoresAsync(const hipExternalSemaphore_t* extSemArray, - const hipExternalSemaphoreWaitParams* paramsArray, - unsigned int numExtSems, hipStream_t stream) -{ - HIP_INIT_API(hipWaitExternalSemaphoresAsync, extSemArray, paramsArray, numExtSems, - stream); - if (extSemArray == nullptr || paramsArray == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - amd::HostQueue* queue = hip::getQueue(stream); - const amd::Device& device = queue->vdev()->device(); - - for (unsigned int i = 0; i < numExtSems; i++) { - if (extSemArray[i] != nullptr) { - amd::ExternalSemaphoreCmd* command = - new amd::ExternalSemaphoreCmd(*queue, extSemArray[i], paramsArray[i].params.fence.value, - amd::ExternalSemaphoreCmd::COMMAND_WAIT_EXTSEMAPHORE); - if (command == nullptr) { - return hipErrorOutOfMemory; - } - command->enqueue(); - command->release(); - } else { - HIP_RETURN(hipErrorInvalidValue); - } - } - HIP_RETURN(hipSuccess); -} - -hipError_t hipDestroyExternalSemaphore(hipExternalSemaphore_t extSem) -{ - HIP_INIT_API(hipDestroyExternalSemaphore, extSem); - if (extSem == nullptr ) { - HIP_RETURN(hipErrorInvalidValue); - } - amd::Device* device = hip::getCurrentDevice()->devices()[0]; - device->DestroyExtSemaphore(extSem); - HIP_RETURN(hipSuccess); -} - - -// ================================================================================================ -hipError_t ihipMalloc(void** ptr, size_t sizeBytes, unsigned int flags) -{ - if (ptr == nullptr) { - return hipErrorInvalidValue; - } - if (sizeBytes == 0) { - *ptr = nullptr; - return hipSuccess; - } - - bool useHostDevice = (flags & CL_MEM_SVM_FINE_GRAIN_BUFFER) != 0; - amd::Context* curDevContext = hip::getCurrentDevice()->asContext(); - amd::Context* amdContext = useHostDevice ? hip::host_device->asContext() : curDevContext; - - if (amdContext == nullptr) { - return hipErrorOutOfMemory; - } - - if (amdContext->devices()[0]->info().maxMemAllocSize_ < sizeBytes) { - return hipErrorOutOfMemory; - } - - *ptr = amd::SvmBuffer::malloc(*amdContext, flags, sizeBytes, amdContext->devices()[0]->info().memBaseAddrAlign_, - useHostDevice ? curDevContext->svmDevices()[0] : nullptr); - if (*ptr == nullptr) { - size_t free = 0, total =0; - hipMemGetInfo(&free, &total); - LogPrintfError("Allocation failed : Device memory : required :%zu | free :%zu | total :%zu \n", sizeBytes, free, total); - return hipErrorOutOfMemory; - } - - return hipSuccess; -} - -hipError_t ihipMemcpy_validate(void* dst, const void* src, size_t sizeBytes, - hipMemcpyKind kind) { - if (dst == nullptr || src == nullptr) { - return hipErrorInvalidValue; - } - size_t sOffset = 0; - amd::Memory* srcMemory = getMemoryObject(src, sOffset); - size_t dOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dst, dOffset); - // Return error if sizeBytes passed to memcpy is more than the actual size allocated - if ((dstMemory && sizeBytes > (dstMemory->getSize() - dOffset)) || - (srcMemory && sizeBytes > (srcMemory->getSize() - sOffset))) { - return hipErrorInvalidValue; - } - return hipSuccess; -} - -hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src, size_t sizeBytes, - hipMemcpyKind kind, amd::HostQueue& queue) { - amd::Command::EventWaitList waitList; - size_t sOffset = 0; - amd::Memory* srcMemory = getMemoryObject(src, sOffset); - size_t dOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dst, dOffset); - amd::Device* queueDevice = &queue.device(); - if ((srcMemory == nullptr) && (dstMemory != nullptr)) { - amd::HostQueue* pQueue = &queue; - if (queueDevice != dstMemory->getContext().devices()[0]) { - pQueue = hip::getNullStream(dstMemory->getContext()); - amd::Command* cmd = queue.getLastQueuedCommand(true); - if (cmd != nullptr) { - waitList.push_back(cmd); - } - } - command = new amd::WriteMemoryCommand(*pQueue, CL_COMMAND_WRITE_BUFFER, waitList, - *dstMemory->asBuffer(), dOffset, sizeBytes, src); - } else if ((srcMemory != nullptr) && (dstMemory == nullptr)) { - amd::HostQueue* pQueue = &queue; - if (queueDevice != srcMemory->getContext().devices()[0]) { - pQueue = hip::getNullStream(srcMemory->getContext()); - amd::Command* cmd = queue.getLastQueuedCommand(true); - if (cmd != nullptr) { - waitList.push_back(cmd); - } - } - command = new amd::ReadMemoryCommand(*pQueue, CL_COMMAND_READ_BUFFER, waitList, - *srcMemory->asBuffer(), sOffset, sizeBytes, dst); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - // Check if the queue device doesn't match the device on any memory object. - // And any of them are not host allocation. - // Hence it's a P2P transfer, because the app has requested access to another GPU - if ((srcMemory->getContext().devices()[0] != dstMemory->getContext().devices()[0]) && - ((srcMemory->getContext().devices().size() == 1) && - (dstMemory->getContext().devices().size() == 1))) { - command = new amd::CopyMemoryP2PCommand(queue, CL_COMMAND_COPY_BUFFER, waitList, - *srcMemory->asBuffer(), *dstMemory->asBuffer(), sOffset, dOffset, sizeBytes); - if (command == nullptr) { - return hipErrorOutOfMemory; - } - // Make sure runtime has valid memory for the command execution. P2P access - // requires page table mapping on the current device to another GPU memory - if (!static_cast(command)->validateMemory()) { - delete command; - return hipErrorInvalidValue; - } - } else { - amd::HostQueue* pQueue = &queue; - if ((srcMemory->getContext().devices()[0] == dstMemory->getContext().devices()[0]) && - (queueDevice != srcMemory->getContext().devices()[0])) { - pQueue = hip::getNullStream(srcMemory->getContext()); - amd::Command* cmd = queue.getLastQueuedCommand(true); - if (cmd != nullptr) { - waitList.push_back(cmd); - } - } else if (srcMemory->getContext().devices()[0] != dstMemory->getContext().devices()[0]) { - // Scenarios such as DtoH where dst is pinned memory - if ((queueDevice != srcMemory->getContext().devices()[0]) && - (dstMemory->getContext().devices().size() != 1)) { - pQueue = hip::getNullStream(srcMemory->getContext()); - amd::Command* cmd = queue.getLastQueuedCommand(true); - if (cmd != nullptr) { - waitList.push_back(cmd); - } - // Scenarios such as HtoD where src is pinned memory - } else if ((queueDevice != dstMemory->getContext().devices()[0]) && - (srcMemory->getContext().devices().size() != 1)) { - pQueue = hip::getNullStream(dstMemory->getContext()); - amd::Command* cmd = queue.getLastQueuedCommand(true); - if (cmd != nullptr) { - waitList.push_back(cmd); - } - } - } - command = new amd::CopyMemoryCommand(*pQueue, CL_COMMAND_COPY_BUFFER, waitList, - *srcMemory->asBuffer(), *dstMemory->asBuffer(), sOffset, dOffset, sizeBytes); - } - } - if (command == nullptr) { - return hipErrorOutOfMemory; - } - if (waitList.size() > 0) { - waitList[0]->release(); - } - return hipSuccess; -} - -// ================================================================================================ -hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind, - amd::HostQueue& queue, bool isAsync = false) { - hipError_t status; - if (sizeBytes == 0) { - // Skip if nothing needs writing. - return hipSuccess; - } - status = ihipMemcpy_validate(dst, src, sizeBytes, kind); - if (status != hipSuccess) { - return status; - } - size_t sOffset = 0; - amd::Memory* srcMemory = getMemoryObject(src, sOffset); - size_t dOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dst, dOffset); - if ((srcMemory == nullptr) && (dstMemory == nullptr)) { - if ((kind == hipMemcpyHostToHost) || (kind == hipMemcpyDefault)) { - queue.finish(); - memcpy(dst, src, sizeBytes); - return hipSuccess; - } else { - return hipErrorInvalidValue; - } - } else if ((srcMemory == nullptr) && (dstMemory != nullptr)) { - isAsync = false; - } else if ((srcMemory != nullptr) && (dstMemory == nullptr)) { - isAsync = false; - } - amd::Command* command = nullptr; - status = ihipMemcpyCommand(command, dst, src, sizeBytes, kind, queue); - if (status != hipSuccess) { - return status; - } - command->enqueue(); - if (!isAsync) { - command->awaitCompletion(); - } else { - amd::HostQueue* newQueue = command->queue(); - if (newQueue != &queue) { - amd::Command::EventWaitList waitList; - amd::Command* cmd = newQueue->getLastQueuedCommand(true); - if (cmd != nullptr) { - waitList.push_back(cmd); - amd::Command* depdentMarker = new amd::Marker(queue, true, waitList); - if (depdentMarker != nullptr) { - depdentMarker->enqueue(); - depdentMarker->release(); - } - cmd->release(); - } - } - } - command->release(); - return hipSuccess; -} - -// ================================================================================================ -hipError_t hipExtMallocWithFlags(void** ptr, size_t sizeBytes, unsigned int flags) { - HIP_INIT_API(hipExtMallocWithFlags, ptr, sizeBytes, flags); - - unsigned int ihipFlags = 0; - if (flags == hipDeviceMallocDefault) { - ihipFlags = 0; - } else if (flags == hipDeviceMallocFinegrained) { - ihipFlags = CL_MEM_SVM_ATOMICS; - } else if (flags == hipMallocSignalMemory) { - ihipFlags = CL_MEM_SVM_ATOMICS | CL_MEM_SVM_FINE_GRAIN_BUFFER | ROCCLR_MEM_HSA_SIGNAL_MEMORY; - if (sizeBytes != 8) { - HIP_RETURN(hipErrorInvalidValue); - } - } else { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(ihipMalloc(ptr, sizeBytes, ihipFlags), (ptr != nullptr)? *ptr : nullptr); -} - -hipError_t hipMalloc(void** ptr, size_t sizeBytes) { - HIP_INIT_API(hipMalloc, ptr, sizeBytes); - - HIP_RETURN_DURATION(ihipMalloc(ptr, sizeBytes, 0), (ptr != nullptr)? *ptr : nullptr); -} - -hipError_t hipHostMalloc(void** ptr, size_t sizeBytes, unsigned int flags) { - HIP_INIT_API(hipHostMalloc, ptr, sizeBytes, flags); - - if (ptr == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - *ptr = nullptr; - - const unsigned int coherentFlags = hipHostMallocCoherent | hipHostMallocNonCoherent; - - // can't have both Coherent and NonCoherent flags set at the same time - if ((flags & coherentFlags) == coherentFlags) { - LogPrintfError( - "Cannot have both coherent and non-coherent flags " - "at the same time, flags: %u coherent flags: %u \n", - flags, coherentFlags); - HIP_RETURN(hipErrorInvalidValue); - } - - unsigned int ihipFlags = CL_MEM_SVM_FINE_GRAIN_BUFFER | (flags << 16); - if (flags == 0 || - flags & (hipHostMallocCoherent | hipHostMallocMapped) || - (!(flags & hipHostMallocNonCoherent) && HIP_HOST_COHERENT)) { - ihipFlags |= CL_MEM_SVM_ATOMICS; - } - - if (flags & hipHostMallocNumaUser) { - ihipFlags |= CL_MEM_FOLLOW_USER_NUMA_POLICY; - } - - HIP_RETURN_DURATION(ihipMalloc(ptr, sizeBytes, ihipFlags), *ptr); -} - -hipError_t hipFree(void* ptr) { - HIP_INIT_API(hipFree, ptr); - - HIP_RETURN(ihipFree(ptr)); -} - -hipError_t hipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpy, dst, src, sizeBytes, kind); - - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN_DURATION(ihipMemcpy(dst, src, sizeBytes, kind, *queue)); -} - -hipError_t hipMemcpyWithStream(void* dst, const void* src, size_t sizeBytes, - hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpyWithStream, dst, src, sizeBytes, kind, stream); - - amd::HostQueue* queue = hip::getQueue(stream); - - HIP_RETURN_DURATION(ihipMemcpy(dst, src, sizeBytes, kind, *queue, false)); -} - -hipError_t hipMemPtrGetInfo(void *ptr, size_t *size) { - HIP_INIT_API(hipMemPtrGetInfo, ptr, size); - - size_t offset = 0; - amd::Memory* svmMem = getMemoryObject(ptr, offset); - - if (svmMem == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - *size = svmMem->getSize(); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipHostFree(void* ptr) { - HIP_INIT_API(hipHostFree, ptr); - - HIP_RETURN(ihipFree(ptr)); -} - -hipError_t ihipArrayDestroy(hipArray* array) { - if (array == nullptr) { - return hipErrorInvalidValue; - } - - cl_mem memObj = reinterpret_cast(array->data); - if (is_valid(memObj) == false) { - return hipErrorInvalidValue; - } - for (auto& dev : g_devices) { - dev->NullStream()->finish(); - } - as_amd(memObj)->release(); - - delete array; - - return hipSuccess; -} - -hipError_t hipFreeArray(hipArray* array) { - HIP_INIT_API(hipFreeArray, array); - - HIP_RETURN(ihipArrayDestroy(array)); -} - -hipError_t hipMemGetAddressRange(hipDeviceptr_t* pbase, size_t* psize, hipDeviceptr_t dptr) { - HIP_INIT_API(hipMemGetAddressRange, pbase, psize, dptr); - - // Since we are using SVM buffer DevicePtr and HostPtr is the same - void* ptr = dptr; - size_t offset = 0; - amd::Memory* svmMem = getMemoryObject(ptr, offset); - - if (svmMem == nullptr) { - HIP_RETURN(hipErrorInvalidDevicePointer); - } - - *pbase = svmMem->getSvmPtr(); - *psize = svmMem->getSize(); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipMemGetInfo(size_t* free, size_t* total) { - HIP_INIT_API(hipMemGetInfo, free, total); - - size_t freeMemory[2]; - amd::Device* device = hip::getCurrentDevice()->devices()[0]; - if(device == nullptr) { - HIP_RETURN(hipErrorInvalidDevice); - } - - if(!device->globalFreeMemory(freeMemory)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *free = freeMemory[0] * Ki; - *total = device->info().globalMemSize_; - - HIP_RETURN(hipSuccess); -} - -hipError_t ihipMallocPitch(void** ptr, size_t* pitch, size_t width, size_t height, size_t depth, - cl_mem_object_type imageType, const cl_image_format* image_format) { - - amd::Device* device = hip::getCurrentDevice()->devices()[0]; - - if (ptr == nullptr) { - return hipErrorInvalidValue; - } - - if ((width == 0) || (height == 0) || (depth == 0)) { - *ptr = nullptr; - return hipSuccess; - } - - const amd::Image::Format imageFormat(*image_format); - - *pitch = amd::alignUp(width * imageFormat.getElementSize(), device->info().imagePitchAlignment_); - - size_t sizeBytes = *pitch * height * depth; - - if (device->info().maxMemAllocSize_ < sizeBytes) { - return hipErrorOutOfMemory; - } - - *ptr = amd::SvmBuffer::malloc(*hip::getCurrentDevice()->asContext(), 0, sizeBytes, - device->info().memBaseAddrAlign_); - - if (*ptr == nullptr) { - return hipErrorOutOfMemory; - } - - return hipSuccess; -} - - -hipError_t hipMallocPitch(void** ptr, size_t* pitch, size_t width, size_t height) { - HIP_INIT_API(hipMallocPitch, ptr, pitch, width, height); - - const cl_image_format image_format = { CL_R, CL_UNSIGNED_INT8 }; - HIP_RETURN(ihipMallocPitch(ptr, pitch, width, height, 1, CL_MEM_OBJECT_IMAGE2D, &image_format), (ptr != nullptr)? *ptr : nullptr); -} - -hipError_t hipMalloc3D(hipPitchedPtr* pitchedDevPtr, hipExtent extent) { - HIP_INIT_API(hipMalloc3D, pitchedDevPtr, extent); - - size_t pitch = 0; - - if (pitchedDevPtr == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - const cl_image_format image_format = { CL_R, CL_UNSIGNED_INT8 }; - hipError_t status = hipSuccess; - status = ihipMallocPitch(&pitchedDevPtr->ptr, &pitch, extent.width, extent.height, extent.depth, - CL_MEM_OBJECT_IMAGE3D, &image_format); - - if (status == hipSuccess) { - pitchedDevPtr->pitch = pitch; - pitchedDevPtr->xsize = extent.width; - pitchedDevPtr->ysize = extent.height; - } - - HIP_RETURN(status, *pitchedDevPtr); -} - -amd::Image* ihipImageCreate(const cl_channel_order channelOrder, - const cl_channel_type channelType, - const cl_mem_object_type imageType, - const size_t imageWidth, - const size_t imageHeight, - const size_t imageDepth, - const size_t imageArraySize, - const size_t imageRowPitch, - const size_t imageSlicePitch, - const uint32_t numMipLevels, - amd::Memory* buffer) { - const amd::Image::Format imageFormat({channelOrder, channelType}); - if (!imageFormat.isValid()) { - LogPrintfError("Invalid Image format for channel Order:%u Type:%u \n", channelOrder, - channelType); - return nullptr; - } - - amd::Context& context = *hip::getCurrentDevice()->asContext(); - if (!imageFormat.isSupported(context, imageType)) { - LogPrintfError("Image type: %u not supported \n", imageType); - return nullptr; - } - - const std::vector& devices = context.devices(); - if (!devices[0]->info().imageSupport_) { - LogPrintfError("Device: 0x%x does not support image \n", devices[0]); - return nullptr; - } - - bool mipMapSupport = true; - for (auto& dev : devices) { - if (!dev->settings().checkExtension(ClKhrMipMapImage)) { - mipMapSupport = false; - } - } - - if (!amd::Image::validateDimensions(devices, - imageType, - imageWidth, - imageHeight, - imageDepth, - imageArraySize)) { - DevLogError("Image does not have valid dimensions \n"); - return nullptr; - } - - if (numMipLevels > 0) { - if (mipMapSupport == true) { - size_t max_dim = std::max(std::max(imageWidth, imageHeight), imageDepth); - size_t mip_levels = 0; - for (mip_levels = 0; max_dim > 0; max_dim >>=1, mip_levels++); - // empty for loop - - if (mip_levels < numMipLevels) { - LogPrintfError("Invalid Mip Levels: %d", numMipLevels); - return nullptr; - } - } else { - LogPrintfError("Mipmap not supported on one of the devices, Mip Level: %d", numMipLevels); - return nullptr; - } - } - - // TODO validate the image descriptor. - - amd::Image* image = nullptr; - if (buffer != nullptr) { - switch (imageType) { - case CL_MEM_OBJECT_IMAGE1D_BUFFER: - case CL_MEM_OBJECT_IMAGE2D: - image = new (context) amd::Image(*buffer->asBuffer(), - imageType, - CL_MEM_READ_WRITE, - imageFormat, - imageWidth, - (imageHeight == 0) ? 1 : imageHeight, - (imageDepth == 0) ? 1 : imageDepth, - imageRowPitch, - imageSlicePitch); - break; - default: - ShouldNotReachHere(); - } - } else { - switch (imageType) { - case CL_MEM_OBJECT_IMAGE1D: - case CL_MEM_OBJECT_IMAGE2D: - case CL_MEM_OBJECT_IMAGE3D: - image = new (context) amd::Image(context, - imageType, - CL_MEM_READ_WRITE, - imageFormat, - imageWidth, - (imageHeight == 0) ? 1 : imageHeight, - (imageDepth == 0) ? 1 : imageDepth, - imageWidth * imageFormat.getElementSize(), /* row pitch */ - imageWidth * imageHeight * imageFormat.getElementSize(), /* slice pitch */ - numMipLevels); - break; - case CL_MEM_OBJECT_IMAGE1D_ARRAY: - image = new (context) amd::Image(context, - imageType, - CL_MEM_READ_WRITE, - imageFormat, - imageWidth, - imageArraySize, - 1, /* image depth */ - imageWidth * imageFormat.getElementSize(), - imageWidth * imageHeight * imageFormat.getElementSize(), - numMipLevels); - break; - case CL_MEM_OBJECT_IMAGE2D_ARRAY: - image = new (context) amd::Image(context, - imageType, - CL_MEM_READ_WRITE, - imageFormat, - imageWidth, - imageHeight, - imageArraySize, - imageWidth * imageFormat.getElementSize(), - imageWidth * imageHeight * imageFormat.getElementSize(), - numMipLevels); - break; - default: - ShouldNotReachHere(); - } - } - - if (image == nullptr) { - return nullptr; - } - - if (!image->create(nullptr)) { - LogPrintfError("Cannot create image: 0x%x \n", image); - delete image; - return nullptr; - } - - return image; -} - -hipError_t ihipArrayCreate(hipArray** array, - const HIP_ARRAY3D_DESCRIPTOR* pAllocateArray, - unsigned int numMipmapLevels) { - // NumChannels specifies the number of packed components per HIP array element; it may be 1, 2, or 4; - if ((pAllocateArray->NumChannels != 1) && - (pAllocateArray->NumChannels != 2) && - (pAllocateArray->NumChannels != 4)) { - return hipErrorInvalidValue; - } - - if ((pAllocateArray->Flags & hipArraySurfaceLoadStore) || - (pAllocateArray->Flags & hipArrayCubemap) || - (pAllocateArray->Flags & hipArrayTextureGather)) { - return hipErrorNotSupported; - } - - const cl_channel_order channelOrder = hip::getCLChannelOrder(pAllocateArray->NumChannels, 0); - const cl_channel_type channelType = hip::getCLChannelType(pAllocateArray->Format, hipReadModeElementType); - const cl_mem_object_type imageType = hip::getCLMemObjectType(pAllocateArray->Width, - pAllocateArray->Height, - pAllocateArray->Depth, - pAllocateArray->Flags); - - amd::Image* image = ihipImageCreate(channelOrder, - channelType, - imageType, - pAllocateArray->Width, - pAllocateArray->Height, - pAllocateArray->Depth, - // The number of layers is determined by the depth extent. - pAllocateArray->Depth, /* array size */ - 0, /* row pitch */ - 0, /* slice pitch */ - numMipmapLevels, - nullptr /* buffer */); - - if (image == nullptr) { - return hipErrorInvalidValue; - } - - cl_mem memObj = as_cl(image); - *array = new hipArray{reinterpret_cast(memObj)}; - - // It is UB to call hipGet*() on an array created via hipArrayCreate()/hipArray3DCreate(). - // This is due to hip not differentiating between runtime and driver types. - // TODO change the hipArray struct in driver_types.h. - (*array)->desc = hip::getChannelFormatDesc(pAllocateArray->NumChannels, pAllocateArray->Format); - (*array)->width = pAllocateArray->Width; - (*array)->height = pAllocateArray->Height; - (*array)->depth = pAllocateArray->Depth; - (*array)->Format = pAllocateArray->Format; - (*array)->NumChannels = pAllocateArray->NumChannels; - - return hipSuccess; -} - -hipError_t hipArrayCreate(hipArray** array, - const HIP_ARRAY_DESCRIPTOR* pAllocateArray) { - HIP_INIT_API(hipArrayCreate, array, pAllocateArray); - - HIP_ARRAY3D_DESCRIPTOR desc = {pAllocateArray->Width, - pAllocateArray->Height, - 0, /* Depth */ - pAllocateArray->Format, - pAllocateArray->NumChannels, - hipArrayDefault /* Flags */}; - - HIP_RETURN(ihipArrayCreate(array, &desc, 0)); -} - - -hipError_t hipMallocArray(hipArray** array, - const hipChannelFormatDesc* desc, - size_t width, - size_t height, - unsigned int flags) { - HIP_INIT_API(hipMallocArray, array, desc, width, height, flags); - - HIP_ARRAY3D_DESCRIPTOR allocateArray = {width, - height, - 0, /* Depth */ - hip::getArrayFormat(*desc), - hip::getNumChannels(*desc), - flags}; - - HIP_RETURN(ihipArrayCreate(array, &allocateArray, 0 /* numMipLevels */)); -} - -hipError_t hipArray3DCreate(hipArray** array, - const HIP_ARRAY3D_DESCRIPTOR* pAllocateArray) { - HIP_INIT_API(hipArray3DCreate, array, pAllocateArray); - - HIP_RETURN(ihipArrayCreate(array, pAllocateArray, 0 /* numMipLevels */)); -} - -hipError_t hipMalloc3DArray(hipArray_t* array, - const hipChannelFormatDesc* desc, - hipExtent extent, - unsigned int flags) { - HIP_INIT_API(hipMalloc3DArray, array, desc, extent, flags); - - HIP_ARRAY3D_DESCRIPTOR allocateArray = {extent.width, - extent.height, - extent.depth, - hip::getArrayFormat(*desc), - hip::getNumChannels(*desc), - flags}; - - HIP_RETURN(ihipArrayCreate(array, &allocateArray, 0)); -} - -hipError_t hipHostGetFlags(unsigned int* flagsPtr, void* hostPtr) { - HIP_INIT_API(hipHostGetFlags, flagsPtr, hostPtr); - - if (flagsPtr == nullptr || hostPtr == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - size_t offset = 0; - amd::Memory* svmMem = getMemoryObject(hostPtr, offset); - - if (svmMem == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - *flagsPtr = svmMem->getMemFlags() >> 16; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) { - HIP_INIT_API(hipHostRegister, hostPtr, sizeBytes, flags); - if(hostPtr != nullptr) { - amd::Memory* mem = new (*hip::host_device->asContext()) amd::Buffer(*hip::host_device->asContext(), CL_MEM_USE_HOST_PTR | CL_MEM_SVM_ATOMICS, sizeBytes); - - constexpr bool sysMemAlloc = false; - constexpr bool skipAlloc = false; - constexpr bool forceAlloc = true; - if (!mem->create(hostPtr, sysMemAlloc, skipAlloc, forceAlloc)) { - mem->release(); - LogPrintfError("Cannot create memory for size: %u with flags: %d \n", sizeBytes, flags); - HIP_RETURN(hipErrorOutOfMemory); - } - - for (const auto& device: hip::getCurrentDevice()->devices()) { - // Since the amd::Memory object is shared between all devices - // it's fine to have multiple addresses mapped to it - const device::Memory* devMem = mem->getDeviceMemory(*device); - amd::MemObjMap::AddMemObj(reinterpret_cast(devMem->virtualAddress()), mem); - } - - amd::MemObjMap::AddMemObj(hostPtr, mem); - HIP_RETURN(hipSuccess); - } else { - HIP_RETURN_DURATION(ihipMalloc(&hostPtr, sizeBytes, flags), hostPtr); - } -} - -hipError_t hipHostUnregister(void* hostPtr) { - HIP_INIT_API(hipHostUnregister, hostPtr); - - for (auto& dev : g_devices) { - dev->NullStream()->finish(); - } - - if (amd::SvmBuffer::malloced(hostPtr)) { - amd::SvmBuffer::free(*hip::host_device->asContext(), hostPtr); - HIP_RETURN(hipSuccess); - } else { - size_t offset = 0; - amd::Memory* mem = getMemoryObject(hostPtr, offset); - - if(mem) { - for (const auto& device: g_devices) { - const device::Memory* devMem = mem->getDeviceMemory(*device->devices()[0]); - if (devMem != nullptr) { - void* vAddr = reinterpret_cast(devMem->virtualAddress()); - if (amd::MemObjMap::FindMemObj(vAddr)) { - amd::MemObjMap::RemoveMemObj(vAddr); - } - } - } - amd::MemObjMap::RemoveMemObj(hostPtr); - mem->release(); - HIP_RETURN(hipSuccess); - } - } - - LogPrintfError("Cannot unregister host_ptr: 0x%x \n", hostPtr); - HIP_RETURN(hipErrorInvalidValue); -} - -// Deprecated function: -hipError_t hipHostAlloc(void** ptr, size_t sizeBytes, unsigned int flags) { - HIP_INIT_API(hipHostAlloc, ptr, sizeBytes, flags); - - HIP_RETURN(ihipMalloc(ptr, sizeBytes, flags), (ptr != nullptr)? *ptr : nullptr); -}; - -inline hipError_t ihipMemcpySymbol_validate(const void* symbol, size_t sizeBytes, size_t offset, size_t &sym_size, hipDeviceptr_t &device_ptr) { - HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(symbol, ihipGetDevice(), &device_ptr, &sym_size)); - - /* Size Check to make sure offset is correct */ - if ((offset + sizeBytes) > sym_size) { - LogPrintfError("Trying to access out of bounds, offset: %u sizeBytes: %u sym_size: %u \n", - offset, sizeBytes, sym_size); - HIP_RETURN(hipErrorInvalidDevicePointer); - } - - device_ptr = reinterpret_cast
(device_ptr) + offset; - return hipSuccess; -} - -hipError_t hipMemcpyToSymbol(const void* symbol, const void* src, size_t sizeBytes, - size_t offset, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyToSymbol, symbol, src, sizeBytes, offset, kind); - - size_t sym_size = 0; - hipDeviceptr_t device_ptr = nullptr; - - hipError_t status = ihipMemcpySymbol_validate(symbol, sizeBytes, offset, sym_size, device_ptr); - if (status != hipSuccess) { - return status; - } - - /* Copy memory from source to destination address */ - HIP_RETURN_DURATION(hipMemcpy(device_ptr, src, sizeBytes, kind)); -} - -hipError_t hipMemcpyFromSymbol(void* dst, const void* symbol, size_t sizeBytes, - size_t offset, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyFromSymbol, symbol, dst, sizeBytes, offset, kind); - - size_t sym_size = 0; - hipDeviceptr_t device_ptr = nullptr; - - hipError_t status = ihipMemcpySymbol_validate(symbol, sizeBytes, offset, sym_size, device_ptr); - if (status != hipSuccess) { - return status; - } - - /* Copy memory from source to destination address */ - HIP_RETURN_DURATION(hipMemcpy(dst, device_ptr, sizeBytes, kind)); -} - -hipError_t hipMemcpyToSymbolAsync(const void* symbol, const void* src, size_t sizeBytes, - size_t offset, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpyToSymbolAsync, symbol, src, sizeBytes, offset, kind, stream); - - STREAM_CAPTURE(hipMemcpyToSymbolAsync, stream, symbol, src, sizeBytes, offset, kind); - - size_t sym_size = 0; - hipDeviceptr_t device_ptr = nullptr; - - hipError_t status = ihipMemcpySymbol_validate(symbol, sizeBytes, offset, sym_size, device_ptr); - if (status != hipSuccess) { - return status; - } - /* Copy memory from source to destination address */ - HIP_RETURN_DURATION(hipMemcpyAsync(device_ptr, src, sizeBytes, kind, stream)); -} - -hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* symbol, size_t sizeBytes, - size_t offset, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpyFromSymbolAsync, symbol, dst, sizeBytes, offset, kind, stream); - - STREAM_CAPTURE(hipMemcpyFromSymbolAsync, stream, dst, symbol, sizeBytes, offset, kind); - - size_t sym_size = 0; - hipDeviceptr_t device_ptr = nullptr; - - hipError_t status = ihipMemcpySymbol_validate(symbol, sizeBytes, offset, sym_size, device_ptr); - if (status != hipSuccess) { - return status; - } - - /* Copy memory from source to destination address */ - HIP_RETURN_DURATION(hipMemcpyAsync(dst, device_ptr, sizeBytes, kind, stream)); -} - -hipError_t hipMemcpyHtoD(hipDeviceptr_t dstDevice, - void* srcHost, - size_t ByteCount) { - HIP_INIT_API(hipMemcpyHtoD, dstDevice, srcHost, ByteCount); - - HIP_RETURN_DURATION(ihipMemcpy(dstDevice, srcHost, ByteCount, hipMemcpyHostToDevice, *hip::getQueue(nullptr))); -} - -hipError_t hipMemcpyDtoH(void* dstHost, - hipDeviceptr_t srcDevice, - size_t ByteCount) { - HIP_INIT_API(hipMemcpyDtoH, dstHost, srcDevice, ByteCount); - - HIP_RETURN_DURATION(ihipMemcpy(dstHost, srcDevice, ByteCount, hipMemcpyDeviceToHost, *hip::getQueue(nullptr))); -} - -hipError_t hipMemcpyDtoD(hipDeviceptr_t dstDevice, - hipDeviceptr_t srcDevice, - size_t ByteCount) { - HIP_INIT_API(hipMemcpyDtoD, dstDevice, srcDevice, ByteCount); - - HIP_RETURN_DURATION(ihipMemcpy(dstDevice, srcDevice, ByteCount, hipMemcpyDeviceToDevice, *hip::getQueue(nullptr))); -} - -hipError_t hipMemcpyAsync(void* dst, const void* src, size_t sizeBytes, - hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpyAsync, dst, src, sizeBytes, kind, stream); - - STREAM_CAPTURE(hipMemcpyAsync, stream, dst, src, sizeBytes, kind); - - amd::HostQueue* queue = hip::getQueue(stream); - - HIP_RETURN_DURATION(ihipMemcpy(dst, src, sizeBytes, kind, *queue, true)); -} - -hipError_t hipMemcpyHtoDAsync(hipDeviceptr_t dstDevice, - void* srcHost, - size_t ByteCount, - hipStream_t stream) { - HIP_INIT_API(hipMemcpyHtoDAsync, dstDevice, srcHost, ByteCount, stream); - - HIP_RETURN_DURATION(ihipMemcpy(dstDevice, srcHost, ByteCount, hipMemcpyHostToDevice, *hip::getQueue(stream), true)); -} - -hipError_t hipMemcpyDtoDAsync(hipDeviceptr_t dstDevice, - hipDeviceptr_t srcDevice, - size_t ByteCount, - hipStream_t stream) { - HIP_INIT_API(hipMemcpyDtoDAsync, dstDevice, srcDevice, ByteCount, stream); - - HIP_RETURN_DURATION(ihipMemcpy(dstDevice, srcDevice, ByteCount, hipMemcpyDeviceToDevice, *hip::getQueue(stream), true)); -} - -hipError_t hipMemcpyDtoHAsync(void* dstHost, - hipDeviceptr_t srcDevice, - size_t ByteCount, - hipStream_t stream) { - HIP_INIT_API(hipMemcpyDtoHAsync, dstHost, srcDevice, ByteCount, stream); - - HIP_RETURN_DURATION(ihipMemcpy(dstHost, srcDevice, ByteCount, hipMemcpyDeviceToHost, *hip::getQueue(stream), true)); -} - -hipError_t ihipMemcpyAtoDCommand(amd::Command*& command, hipArray* srcArray, void* dstDevice, - amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D copyRegion, size_t dstRowPitch, size_t dstSlicePitch, - amd::HostQueue* queue) { - size_t dstOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dstDevice, dstOffset); - if (srcArray == nullptr || (dstMemory == nullptr)) { - return hipErrorInvalidValue; - } - cl_mem srcMemObj = reinterpret_cast(srcArray->data); - if (!is_valid(srcMemObj)) { - return hipErrorInvalidValue; - } - - amd::Image* srcImage = as_amd(srcMemObj)->asImage(); - // HIP assumes the width is in bytes, but OCL assumes it's in pixels. - const size_t elementSize = srcImage->getImageFormat().getElementSize(); - static_cast(srcOrigin)[0] /= elementSize; - static_cast(copyRegion)[0] /= elementSize; - - amd::BufferRect srcRect; - if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), - srcImage->getRowPitch(), srcImage->getSlicePitch())) { - return hipErrorInvalidValue; - } - - amd::BufferRect dstRect; - if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), - dstRowPitch, dstSlicePitch)) { - return hipErrorInvalidValue; - } - dstRect.start_ += dstOffset; - dstRect.end_ += dstOffset; - - const size_t copySizeInBytes = - copyRegion[0] * copyRegion[1] * copyRegion[2] * srcImage->getImageFormat().getElementSize(); - if (!srcImage->validateRegion(srcOrigin, copyRegion) || - !dstMemory->validateRegion(dstOrigin, {copySizeInBytes, 0, 0})) { - return hipErrorInvalidValue; - } - - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_IMAGE_TO_BUFFER, - amd::Command::EventWaitList{}, *srcImage, *dstMemory, - srcOrigin, dstOrigin, copyRegion, srcRect, dstRect); - - if (command == nullptr) { - return hipErrorOutOfMemory; - } - return hipSuccess; -} - -hipError_t ihipMemcpyDtoACommand(amd::Command*& command, void* srcDevice, hipArray* dstArray, - amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D copyRegion, size_t srcRowPitch, size_t srcSlicePitch, - amd::HostQueue* queue) { - size_t srcOffset = 0; - amd::Memory* srcMemory = getMemoryObject(srcDevice, srcOffset); - if ((srcMemory == nullptr) || dstArray == nullptr) { - return hipErrorInvalidValue; - } - cl_mem dstMemObj = reinterpret_cast(dstArray->data); - if (!is_valid(dstMemObj)) { - return hipErrorInvalidValue; - } - - amd::Image* dstImage = as_amd(dstMemObj)->asImage(); - // HIP assumes the width is in bytes, but OCL assumes it's in pixels. - const size_t elementSize = dstImage->getImageFormat().getElementSize(); - static_cast(dstOrigin)[0] /= elementSize; - static_cast(copyRegion)[0] /= elementSize; - - amd::BufferRect srcRect; - if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), - srcRowPitch, srcSlicePitch)) { - return hipErrorInvalidValue; - } - srcRect.start_ += srcOffset; - srcRect.end_ += srcOffset; - - amd::BufferRect dstRect; - if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), - dstImage->getRowPitch(), dstImage->getSlicePitch())) { - return hipErrorInvalidValue; - } - - const size_t copySizeInBytes = - copyRegion[0] * copyRegion[1] * copyRegion[2] * dstImage->getImageFormat().getElementSize(); - if (!srcMemory->validateRegion(srcOrigin, {copySizeInBytes, 0, 0}) || - !dstImage->validateRegion(dstOrigin, copyRegion)) { - return hipErrorInvalidValue; - } - - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_BUFFER_TO_IMAGE, - amd::Command::EventWaitList{}, *srcMemory, *dstImage, - srcOrigin, dstOrigin, copyRegion, srcRect, dstRect); - - if (command == nullptr) { - return hipErrorOutOfMemory; - } - return hipSuccess; -} - -hipError_t ihipMemcpyDtoDCommand(amd::Command*& command, void* srcDevice, void* dstDevice, - amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D copyRegion, size_t srcRowPitch, size_t srcSlicePitch, - size_t dstRowPitch, size_t dstSlicePitch, amd::HostQueue* queue) { - size_t srcOffset = 0; - amd::Memory* srcMemory = getMemoryObject(srcDevice, srcOffset); - size_t dstOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dstDevice, dstOffset); - - if ((srcMemory == nullptr) || (dstMemory == nullptr)) { - return hipErrorInvalidValue; - } - - amd::BufferRect srcRect; - if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), - srcRowPitch, srcSlicePitch)) { - return hipErrorInvalidValue; - } - srcRect.start_ += srcOffset; - srcRect.end_ += srcOffset; - - amd::Coord3D srcStart(srcRect.start_, 0, 0); - amd::Coord3D srcSize(srcRect.end_ - srcRect.start_, 1, 1); - if (!srcMemory->validateRegion(srcStart, srcSize)) { - return hipErrorInvalidValue; - } - - amd::BufferRect dstRect; - if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), - dstRowPitch, dstSlicePitch)) { - return hipErrorInvalidValue; - } - dstRect.start_ += dstOffset; - dstRect.end_ += dstOffset; - - amd::Coord3D dstStart(dstRect.start_, 0, 0); - amd::Coord3D dstSize(dstRect.end_ - dstRect.start_, 1, 1); - if (!dstMemory->validateRegion(dstStart, dstSize)) { - return hipErrorInvalidValue; - } - - amd::CopyMemoryCommand* copyCommand = new amd::CopyMemoryCommand( - *queue, CL_COMMAND_COPY_BUFFER_RECT, amd::Command::EventWaitList{}, *srcMemory, *dstMemory, - srcStart, dstStart, copyRegion, srcRect, dstRect); - - if (copyCommand == nullptr) { - return hipErrorOutOfMemory; - } - - if (!copyCommand->validatePeerMemory()) { - delete copyCommand; - return hipErrorInvalidValue; - } - command = copyCommand; - return hipSuccess; -} - -hipError_t ihipMemcpyDtoHCommand(amd::Command*& command, void* srcDevice, void* dstHost, - amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D copyRegion, size_t srcRowPitch, size_t srcSlicePitch, - size_t dstRowPitch, size_t dstSlicePitch, amd::HostQueue* queue) { - size_t srcOffset = 0; - amd::Memory* srcMemory = getMemoryObject(srcDevice, srcOffset); - - if ((srcMemory == nullptr) || (dstHost == nullptr)) { - return hipErrorInvalidValue; - } - - amd::BufferRect srcRect; - if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), - srcRowPitch, srcSlicePitch)) { - return hipErrorInvalidValue; - } - srcRect.start_ += srcOffset; - srcRect.end_ += srcOffset; - - amd::Coord3D srcStart(srcRect.start_, 0, 0); - amd::Coord3D srcSize(srcRect.end_ - srcRect.start_, 1, 1); - if (!srcMemory->validateRegion(srcStart, srcSize)) { - return hipErrorInvalidValue; - } - - amd::BufferRect dstRect; - if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), - dstRowPitch, dstSlicePitch)) { - return hipErrorInvalidValue; - } - - amd::ReadMemoryCommand* readCommand = - new amd::ReadMemoryCommand(*queue, CL_COMMAND_READ_BUFFER_RECT, amd::Command::EventWaitList{}, - *srcMemory, srcStart, copyRegion, dstHost, srcRect, dstRect); - - if (readCommand == nullptr) { - return hipErrorOutOfMemory; - } - - if (!readCommand->validatePeerMemory()) { - delete readCommand; - return hipErrorInvalidValue; - } - command = readCommand; - return hipSuccess; -} - -hipError_t ihipMemcpyHtoDCommand(amd::Command*& command, const void* srcHost, void* dstDevice, - amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D copyRegion, size_t srcRowPitch, size_t srcSlicePitch, - size_t dstRowPitch, size_t dstSlicePitch, amd::HostQueue* queue) { - size_t dstOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dstDevice, dstOffset); - - if ((srcHost == nullptr) || (dstMemory == nullptr)) { - return hipErrorInvalidValue; - } - - amd::BufferRect srcRect; - if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), - srcRowPitch, srcSlicePitch)) { - return hipErrorInvalidValue; - } - - amd::BufferRect dstRect; - if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), - dstRowPitch, dstSlicePitch)) { - return hipErrorInvalidValue; - } - dstRect.start_ += dstOffset; - dstRect.end_ += dstOffset; - - amd::Coord3D dstStart(dstRect.start_, 0, 0); - amd::Coord3D dstSize(dstRect.end_ - dstRect.start_, 1, 1); - if (!dstMemory->validateRegion(dstStart, dstSize)) { - return hipErrorInvalidValue; - } - - amd::WriteMemoryCommand* writeCommand = new amd::WriteMemoryCommand( - *queue, CL_COMMAND_WRITE_BUFFER_RECT, amd::Command::EventWaitList{}, *dstMemory, dstStart, - copyRegion, srcHost, dstRect, srcRect); - - if (writeCommand == nullptr) { - return hipErrorOutOfMemory; - } - - if (!writeCommand->validatePeerMemory()) { - delete writeCommand; - return hipErrorInvalidValue; - } - command = writeCommand; - return hipSuccess; -} - -hipError_t ihipMemcpyHtoH(const void* srcHost, void* dstHost, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, size_t srcRowPitch, - size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch) { - if ((srcHost == nullptr) || (dstHost == nullptr)) { - return hipErrorInvalidValue; - } - - amd::BufferRect srcRect; - if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), - srcRowPitch, srcSlicePitch)) { - return hipErrorInvalidValue; - } - - amd::BufferRect dstRect; - if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), - dstRowPitch, dstSlicePitch)) { - return hipErrorInvalidValue; - } - - for (size_t slice = 0; slice < copyRegion[2]; slice++) { - for (size_t row = 0; row < copyRegion[1]; row++) { - const void* srcRow = static_cast(srcHost) + srcRect.start_ + - row * srcRect.rowPitch_ + slice * srcRect.slicePitch_; - void* dstRow = static_cast(dstHost) + dstRect.start_ + row * dstRect.rowPitch_ + - slice * dstRect.slicePitch_; - std::memcpy(dstRow, srcRow, copyRegion[0]); - } - } - - return hipSuccess; -} - -hipError_t ihipMemcpyAtoACommand(amd::Command*& command, hipArray* srcArray, hipArray* dstArray, - amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D copyRegion, amd::HostQueue* queue) { - if (dstArray == nullptr || srcArray == nullptr) { - return hipErrorInvalidValue; - } - cl_mem srcMemObj = reinterpret_cast(srcArray->data); - cl_mem dstMemObj = reinterpret_cast(dstArray->data); - if (!is_valid(srcMemObj) || !is_valid(dstMemObj)) { - return hipErrorInvalidValue; - } - - amd::Image* srcImage = as_amd(srcMemObj)->asImage(); - amd::Image* dstImage = as_amd(dstMemObj)->asImage(); - - // HIP assumes the width is in bytes, but OCL assumes it's in pixels. - // Note that src and dst should have the same element size. - assert(srcImage->getImageFormat().getElementSize() == - dstImage->getImageFormat().getElementSize()); - const size_t elementSize = srcImage->getImageFormat().getElementSize(); - static_cast(srcOrigin)[0] /= elementSize; - static_cast(dstOrigin)[0] /= elementSize; - static_cast(copyRegion)[0] /= elementSize; - - if (!srcImage->validateRegion(srcOrigin, copyRegion) || - !dstImage->validateRegion(dstOrigin, copyRegion)) { - return hipErrorInvalidValue; - } - - command = - new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_IMAGE, amd::Command::EventWaitList{}, - *srcImage, *dstImage, srcOrigin, dstOrigin, copyRegion); - - if (command == nullptr) { - return hipErrorOutOfMemory; - } - return hipSuccess; -} - -hipError_t ihipMemcpyHtoACommand(amd::Command*& command, const void* srcHost, hipArray* dstArray, - amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D copyRegion, size_t srcRowPitch, size_t srcSlicePitch, - amd::HostQueue* queue) { - if ((srcHost == nullptr) || dstArray == nullptr) { - return hipErrorInvalidValue; - } - cl_mem dstMemObj = reinterpret_cast(dstArray->data); - if (!is_valid(dstMemObj)) { - return hipErrorInvalidValue; - } - - amd::BufferRect srcRect; - if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), - srcRowPitch, srcSlicePitch)) { - return hipErrorInvalidValue; - } - - amd::Image* dstImage = as_amd(dstMemObj)->asImage(); - // HIP assumes the width is in bytes, but OCL assumes it's in pixels. - const size_t elementSize = dstImage->getImageFormat().getElementSize(); - static_cast(dstOrigin)[0] /= elementSize; - static_cast(copyRegion)[0] /= elementSize; - - if (!dstImage->validateRegion(dstOrigin, copyRegion)) { - return hipErrorInvalidValue; - } - - command = new amd::WriteMemoryCommand( - *queue, CL_COMMAND_WRITE_IMAGE, amd::Command::EventWaitList{}, *dstImage, dstOrigin, - copyRegion, static_cast(srcHost) + srcRect.start_, srcRowPitch, srcSlicePitch); - - if (command == nullptr) { - return hipErrorOutOfMemory; - } - return hipSuccess; -} - -hipError_t ihipMemcpyAtoHCommand(amd::Command*& command, hipArray* srcArray, void* dstHost, - amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D copyRegion, size_t dstRowPitch, size_t dstSlicePitch, - amd::HostQueue* queue) { - if (srcArray == nullptr || (dstHost == nullptr)) { - return hipErrorInvalidValue; - } - cl_mem srcMemObj = reinterpret_cast(srcArray->data); - if (!is_valid(srcMemObj)) { - return hipErrorInvalidValue; - } - - amd::BufferRect dstRect; - if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), - dstRowPitch, dstSlicePitch)) { - return hipErrorInvalidValue; - } - - amd::Image* srcImage = as_amd(srcMemObj)->asImage(); - // HIP assumes the width is in bytes, but OCL assumes it's in pixels. - const size_t elementSize = srcImage->getImageFormat().getElementSize(); - static_cast(srcOrigin)[0] /= elementSize; - static_cast(copyRegion)[0] /= elementSize; - - if (!srcImage->validateRegion(srcOrigin, copyRegion) || - !srcImage->isRowSliceValid(dstRowPitch, dstSlicePitch, copyRegion[0], copyRegion[1])) { - return hipErrorInvalidValue; - } - - command = new amd::ReadMemoryCommand( - *queue, CL_COMMAND_READ_IMAGE, amd::Command::EventWaitList{}, *srcImage, srcOrigin, - copyRegion, static_cast(dstHost) + dstRect.start_, dstRowPitch, dstSlicePitch); - - if (command == nullptr) { - return hipErrorOutOfMemory; - } - return hipSuccess; -} - -hipError_t ihipGetMemcpyParam3DCommand(amd::Command*& command, const HIP_MEMCPY3D* pCopy, - amd::HostQueue* queue) { - // If {src/dst}MemoryType is hipMemoryTypeUnified, {src/dst}Device and {src/dst}Pitch specify the - // (unified virtual address space) base address of the source data and the bytes per row to apply. - // {src/dst}Array is ignored. - hipMemoryType srcMemoryType = pCopy->srcMemoryType; - if (srcMemoryType == hipMemoryTypeUnified) { - srcMemoryType = - amd::MemObjMap::FindMemObj(pCopy->srcDevice) ? hipMemoryTypeDevice : hipMemoryTypeHost; - if (srcMemoryType == hipMemoryTypeHost) { - // {src/dst}Host may be unitialized. Copy over {src/dst}Device into it if we detect system - // memory. - const_cast(pCopy)->srcHost = pCopy->srcDevice; - } - } - hipMemoryType dstMemoryType = pCopy->dstMemoryType; - if (dstMemoryType == hipMemoryTypeUnified) { - dstMemoryType = - amd::MemObjMap::FindMemObj(pCopy->dstDevice) ? hipMemoryTypeDevice : hipMemoryTypeHost; - if (srcMemoryType == hipMemoryTypeHost) { - const_cast(pCopy)->dstHost = pCopy->dstDevice; - } - } - - // If {src/dst}MemoryType is hipMemoryTypeHost, check if the memory was prepinned. - // In that case upgrade the copy type to hipMemoryTypeDevice to avoid extra pinning. - if (srcMemoryType == hipMemoryTypeHost) { - amd::Memory* mem = amd::MemObjMap::FindMemObj(pCopy->srcHost); - srcMemoryType = mem ? hipMemoryTypeDevice : hipMemoryTypeHost; - if (srcMemoryType == hipMemoryTypeDevice) { - const_cast(pCopy)->srcDevice = const_cast(pCopy->srcHost); - } - } - if (dstMemoryType == hipMemoryTypeHost) { - amd::Memory* mem = amd::MemObjMap::FindMemObj(pCopy->dstHost); - dstMemoryType = mem ? hipMemoryTypeDevice : hipMemoryTypeHost; - if (dstMemoryType == hipMemoryTypeDevice) { - const_cast(pCopy)->dstDevice = const_cast(pCopy->dstDevice); - } - } - - amd::Coord3D srcOrigin = {pCopy->srcXInBytes, pCopy->srcY, pCopy->srcZ}; - amd::Coord3D dstOrigin = {pCopy->dstXInBytes, pCopy->dstY, pCopy->dstZ}; - amd::Coord3D copyRegion = {pCopy->WidthInBytes, pCopy->Height, pCopy->Depth}; - - if ((srcMemoryType == hipMemoryTypeHost) && (dstMemoryType == hipMemoryTypeDevice)) { - // Host to Device. - return ihipMemcpyHtoDCommand(command, pCopy->srcHost, pCopy->dstDevice, srcOrigin, dstOrigin, - copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, - pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, queue); - } else if ((srcMemoryType == hipMemoryTypeDevice) && (dstMemoryType == hipMemoryTypeHost)) { - // Device to Host. - return ihipMemcpyDtoHCommand(command, pCopy->srcDevice, pCopy->dstHost, srcOrigin, dstOrigin, - copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, - pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, queue); - } else if ((srcMemoryType == hipMemoryTypeDevice) && (dstMemoryType == hipMemoryTypeDevice)) { - // Device to Device. - return ihipMemcpyDtoDCommand(command, pCopy->srcDevice, pCopy->dstDevice, srcOrigin, dstOrigin, - copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, - pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, queue); - } else if ((srcMemoryType == hipMemoryTypeHost) && (dstMemoryType == hipMemoryTypeArray)) { - // Host to Image. - return ihipMemcpyHtoACommand(command, pCopy->srcHost, pCopy->dstArray, srcOrigin, dstOrigin, - copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, - queue); - } else if ((srcMemoryType == hipMemoryTypeArray) && (dstMemoryType == hipMemoryTypeHost)) { - // Image to Host. - return ihipMemcpyAtoHCommand(command, pCopy->srcArray, pCopy->dstHost, srcOrigin, dstOrigin, - copyRegion, pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, - queue); - } else if ((srcMemoryType == hipMemoryTypeDevice) && (dstMemoryType == hipMemoryTypeArray)) { - // Device to Image. - return ihipMemcpyDtoACommand(command, pCopy->srcDevice, pCopy->dstArray, srcOrigin, dstOrigin, - copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, - queue); - } else if ((srcMemoryType == hipMemoryTypeArray) && (dstMemoryType == hipMemoryTypeDevice)) { - // Image to Device. - return ihipMemcpyAtoDCommand(command, pCopy->srcArray, pCopy->dstDevice, srcOrigin, dstOrigin, - copyRegion, pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, - queue); - } else if ((srcMemoryType == hipMemoryTypeArray) && (dstMemoryType == hipMemoryTypeArray)) { - // Image to Image. - return ihipMemcpyAtoACommand(command, pCopy->srcArray, pCopy->dstArray, srcOrigin, dstOrigin, - copyRegion, queue); - } else { - ShouldNotReachHere(); - } - - return hipSuccess; -} - -inline hipError_t ihipMemcpyCmdEnqueue(amd::Command* command, bool isAsync = false) { - hipError_t status = hipSuccess; - if (command == nullptr) { - return hipErrorOutOfMemory; - } - command->enqueue(); - if (!isAsync) { - if (!command->awaitCompletion()) { - status = hipErrorUnknown; - } - } - command->release(); - return status; -} - -hipError_t ihipMemcpyParam3D(const HIP_MEMCPY3D* pCopy, hipStream_t stream, bool isAsync = false) { - amd::Command* command; - hipError_t status; - if (pCopy->WidthInBytes == 0 || pCopy->Height == 0 || pCopy->Depth == 0) { - LogPrintfInfo("Either Width :%d or Height: %d and Depth: %d is zero", pCopy->WidthInBytes, - pCopy->Height, pCopy->Depth); - return hipSuccess; - } - // If {src/dst}MemoryType is hipMemoryTypeUnified, {src/dst}Device and {src/dst}Pitch specify the (unified virtual address space) - // base address of the source data and the bytes per row to apply. {src/dst}Array is ignored. - hipMemoryType srcMemoryType = pCopy->srcMemoryType; - if (srcMemoryType == hipMemoryTypeUnified) { - srcMemoryType = amd::MemObjMap::FindMemObj(pCopy->srcDevice) ? hipMemoryTypeDevice : hipMemoryTypeHost; - if (srcMemoryType == hipMemoryTypeHost) { - // {src/dst}Host may be unitialized. Copy over {src/dst}Device into it if we detect system memory. - const_cast(pCopy)->srcHost = pCopy->srcDevice; - } - } - hipMemoryType dstMemoryType = pCopy->dstMemoryType; - if (dstMemoryType == hipMemoryTypeUnified) { - dstMemoryType = amd::MemObjMap::FindMemObj(pCopy->dstDevice) ? hipMemoryTypeDevice : hipMemoryTypeHost; - if (srcMemoryType == hipMemoryTypeHost) { - const_cast(pCopy)->dstHost = pCopy->dstDevice; - } - } - // If {src/dst}MemoryType is hipMemoryTypeHost, check if the memory was prepinned. - // In that case upgrade the copy type to hipMemoryTypeDevice to avoid extra pinning. - if (srcMemoryType == hipMemoryTypeHost) { - amd::Memory* mem = amd::MemObjMap::FindMemObj(pCopy->srcHost); - srcMemoryType = mem ? hipMemoryTypeDevice : hipMemoryTypeHost; - } - if (dstMemoryType == hipMemoryTypeHost) { - amd::Memory* mem = amd::MemObjMap::FindMemObj(pCopy->dstHost); - dstMemoryType = mem ? hipMemoryTypeDevice : hipMemoryTypeHost; - } - if ((srcMemoryType == hipMemoryTypeHost) && (dstMemoryType == hipMemoryTypeHost)) { - amd::Coord3D srcOrigin = {pCopy->srcXInBytes, pCopy->srcY, pCopy->srcZ}; - amd::Coord3D dstOrigin = {pCopy->dstXInBytes, pCopy->dstY, pCopy->dstZ}; - amd::Coord3D copyRegion = {pCopy->WidthInBytes, (pCopy->Height != 0) ? pCopy->Height : 1, - (pCopy->Depth != 0) ? pCopy->Depth : 1}; - // Host to Host. - return ihipMemcpyHtoH(pCopy->srcHost, pCopy->dstHost, srcOrigin, dstOrigin, copyRegion, - pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, pCopy->dstPitch, - pCopy->dstPitch * pCopy->dstHeight); - } else { - status = ihipGetMemcpyParam3DCommand(command, pCopy, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); - } -} - -hipError_t ihipMemcpyParam2D(const hip_Memcpy2D* pCopy, - hipStream_t stream, - bool isAsync = false) { - HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*pCopy); - - return ihipMemcpyParam3D(&desc, stream, isAsync); -} - -hipError_t ihipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, - size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { - hip_Memcpy2D desc = {}; - if (spitch == 0 || dpitch == 0) { - return hipErrorUnknown; - } - if (width == 0 || height == 0) { - return hipSuccess; - } - - desc.srcXInBytes = 0; - desc.srcY = 0; - desc.srcMemoryType = std::get<0>(hip::getMemoryType(kind)); - desc.srcHost = src; - desc.srcDevice = const_cast(src); - desc.srcArray = nullptr; // Ignored. - desc.srcPitch = spitch; - - desc.dstXInBytes = 0; - desc.dstY = 0; - desc.dstMemoryType = std::get<1>(hip::getMemoryType(kind)); - desc.dstHost = dst; - desc.dstDevice = dst; - desc.dstArray = nullptr; // Ignored. - desc.dstPitch = dpitch; - - desc.WidthInBytes = width; - desc.Height = height; - - return ihipMemcpyParam2D(&desc, stream, isAsync); -} - -hipError_t hipMemcpyParam2D(const hip_Memcpy2D* pCopy) { - HIP_INIT_API(hipMemcpyParam2D, pCopy); - - HIP_RETURN_DURATION(ihipMemcpyParam2D(pCopy, nullptr)); -} - -hipError_t hipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, - size_t height, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpy2D, dst, dpitch, src, spitch, width, height, kind); - - HIP_RETURN_DURATION(ihipMemcpy2D(dst, dpitch, src, spitch, width, height, kind, nullptr)); -} - -hipError_t hipMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, - size_t height, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpy2DAsync, dst, dpitch, src, spitch, width, height, kind, stream); - - HIP_RETURN_DURATION(ihipMemcpy2D(dst, dpitch, src, spitch, width, height, kind, stream, true)); -} - -hipError_t ihipMemcpy2DToArray(hipArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { - if (dst == nullptr) { - HIP_RETURN(hipErrorInvalidResourceHandle); - } - - hip_Memcpy2D desc = {}; - - desc.srcXInBytes = 0; - desc.srcY = 0; - desc.srcMemoryType = std::get<0>(hip::getMemoryType(kind)); - desc.srcHost = const_cast(src); - desc.srcDevice = const_cast(src); - desc.srcArray = nullptr; - desc.srcPitch = spitch; - - desc.dstXInBytes = wOffset; - desc.dstY = hOffset; - desc.dstMemoryType = hipMemoryTypeArray; - desc.dstHost = nullptr; - desc.dstDevice = nullptr; - desc.dstArray = dst; - desc.dstPitch = 0; // Ignored. - - desc.WidthInBytes = width; - desc.Height = height; - - return ihipMemcpyParam2D(&desc, stream, isAsync); -} - -hipError_t hipMemcpy2DToArray(hipArray* dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpy2DToArray, dst, wOffset, hOffset, src, spitch, width, height, kind); - - if (spitch == 0) { - HIP_RETURN(hipErrorInvalidPitchValue); - } - - HIP_RETURN_DURATION(ihipMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind, nullptr)); -} - -hipError_t hipMemcpyToArray(hipArray* dst, size_t wOffset, size_t hOffset, const void* src, size_t count, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyToArray, dst, wOffset, hOffset, src, count, kind); - - if (dst == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - const size_t arrayHeight = (dst->height != 0) ? dst->height : 1; - const size_t witdthInBytes = count / arrayHeight; - - const size_t height = (count / dst->width) / hip::getElementSize(dst); - - HIP_RETURN_DURATION(ihipMemcpy2DToArray(dst, wOffset, hOffset, src, 0 /* spitch */, witdthInBytes, height, kind, nullptr)); -} - -hipError_t ihipMemcpy2DFromArray(void* dst, size_t dpitch, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { - if (src == nullptr) { - HIP_RETURN(hipErrorInvalidResourceHandle); - } - - hip_Memcpy2D desc = {}; - - desc.srcXInBytes = wOffsetSrc; - desc.srcY = hOffsetSrc; - desc.srcMemoryType = hipMemoryTypeArray; - desc.srcHost = nullptr; - desc.srcDevice = nullptr; - desc.srcArray = const_cast(src); - desc.srcPitch = 0; // Ignored. - - desc.dstXInBytes = 0; - desc.dstY = 0; - desc.dstMemoryType = std::get<1>(hip::getMemoryType(kind)); - desc.dstHost = dst; - desc.dstDevice = dst; - desc.dstArray = nullptr; - desc.dstPitch = dpitch; - - desc.WidthInBytes = width; - desc.Height = height; - - return ihipMemcpyParam2D(&desc, stream, isAsync); -} - -hipError_t hipMemcpyFromArray(void* dst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffset, size_t count, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyFromArray, dst, src, wOffsetSrc, hOffset, count, kind); - - if (src == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - const size_t arrayHeight = (src->height != 0) ? src->height : 1; - const size_t witdthInBytes = count / arrayHeight; - - const size_t height = (count / src->width) / hip::getElementSize(src); - - HIP_RETURN_DURATION(ihipMemcpy2DFromArray(dst, 0 /* dpitch */, src, wOffsetSrc, hOffset, witdthInBytes, height, kind, nullptr)); -} - -hipError_t ihipMemcpyAtoD(hipArray* srcArray, void* dstDevice, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, size_t dstRowPitch, - size_t dstSlicePitch, hipStream_t stream, bool isAsync = false) { - amd::Command* command; - hipError_t status = - ihipMemcpyAtoDCommand(command, srcArray, dstDevice, srcOrigin, dstOrigin, copyRegion, - dstRowPitch, dstSlicePitch, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); -} -hipError_t ihipMemcpyDtoA(void* srcDevice, hipArray* dstArray, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, size_t srcRowPitch, - size_t srcSlicePitch, hipStream_t stream, bool isAsync = false) { - amd::Command* command; - hipError_t status = - ihipMemcpyDtoACommand(command, srcDevice, dstArray, srcOrigin, dstOrigin, copyRegion, - srcRowPitch, srcSlicePitch, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); -} -hipError_t ihipMemcpyDtoD(void* srcDevice, void* dstDevice, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, size_t srcRowPitch, - size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, - hipStream_t stream, bool isAsync = false) { - amd::Command* command; - hipError_t status = ihipMemcpyDtoDCommand(command, srcDevice, dstDevice, srcOrigin, dstOrigin, - copyRegion, srcRowPitch, srcSlicePitch, dstRowPitch, - dstSlicePitch, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); -} -hipError_t ihipMemcpyDtoH(void* srcDevice, void* dstHost, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, size_t srcRowPitch, - size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, - hipStream_t stream, bool isAsync = false) { - amd::Command* command; - hipError_t status = ihipMemcpyDtoHCommand(command, srcDevice, dstHost, srcOrigin, dstOrigin, - copyRegion, srcRowPitch, srcSlicePitch, dstRowPitch, - dstSlicePitch, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); -} -hipError_t ihipMemcpyHtoD(const void* srcHost, void* dstDevice, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, size_t srcRowPitch, - size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, - hipStream_t stream, bool isAsync = false) { - amd::Command* command; - hipError_t status = ihipMemcpyHtoDCommand(command, srcHost, dstDevice, srcOrigin, dstOrigin, - copyRegion, srcRowPitch, srcSlicePitch, dstRowPitch, - dstSlicePitch, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); -} -hipError_t ihipMemcpyAtoA(hipArray* srcArray, hipArray* dstArray, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, hipStream_t stream, - bool isAsync = false) { - amd::Command* command; - hipError_t status = ihipMemcpyAtoACommand(command, srcArray, dstArray, srcOrigin, dstOrigin, - copyRegion, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); -} -hipError_t ihipMemcpyHtoA(const void* srcHost, hipArray* dstArray, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, size_t srcRowPitch, - size_t srcSlicePitch, hipStream_t stream, bool isAsync = false) { - amd::Command* command; - hipError_t status = - ihipMemcpyHtoACommand(command, srcHost, dstArray, srcOrigin, dstOrigin, copyRegion, - srcRowPitch, srcSlicePitch, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); -} -hipError_t ihipMemcpyAtoH(hipArray* srcArray, void* dstHost, amd::Coord3D srcOrigin, - amd::Coord3D dstOrigin, amd::Coord3D copyRegion, size_t dstRowPitch, - size_t dstSlicePitch, hipStream_t stream, bool isAsync = false) { - amd::Command* command; - hipError_t status = - ihipMemcpyAtoHCommand(command, srcArray, dstHost, srcOrigin, dstOrigin, copyRegion, - dstRowPitch, dstSlicePitch, hip::getQueue(stream)); - if (status != hipSuccess) return status; - return ihipMemcpyCmdEnqueue(command, isAsync); -} - -hipError_t hipMemcpyHtoA(hipArray* dstArray, - size_t dstOffset, - const void* srcHost, - size_t ByteCount) { - HIP_INIT_API(hipMemcpyHtoA, dstArray, dstOffset, srcHost, ByteCount); - - HIP_RETURN_DURATION(ihipMemcpyHtoA(srcHost, dstArray, {0, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, 0, 0, nullptr)); -} - -hipError_t hipMemcpyAtoH(void* dstHost, - hipArray* srcArray, - size_t srcOffset, - size_t ByteCount) { - HIP_INIT_API(hipMemcpyAtoH, dstHost, srcArray, srcOffset, ByteCount); - - HIP_RETURN_DURATION(ihipMemcpyAtoH(srcArray, dstHost, {srcOffset, 0, 0}, {0, 0, 0}, {ByteCount, 1, 1}, 0, 0, nullptr)); -} - -hipError_t ihipMemcpy3D_validate(const hipMemcpy3DParms* p) { - // The struct passed to hipMemcpy3D() must specify one of srcArray or srcPtr and one of dstArray - // or dstPtr. Passing more than one non-zero source or destination will cause hipMemcpy3D() to - // return an error. - if (p == nullptr || ((p->srcArray != nullptr) && (p->srcPtr.ptr != nullptr)) || - ((p->dstArray != nullptr) && (p->dstPtr.ptr != nullptr))) { - return hipErrorInvalidValue; - } - - // If the source and destination are both arrays, hipMemcpy3D() will return an error if they do - // not have the same element size. - if (((p->srcArray != nullptr) && (p->dstArray != nullptr)) && - (hip::getElementSize(p->dstArray) != hip::getElementSize(p->dstArray))) { - return hipErrorInvalidValue; - } - return hipSuccess; -} - -hipError_t ihipMemcpy3DCommand(amd::Command*& command, const hipMemcpy3DParms* p, - amd::HostQueue* queue) { - const HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*p); - return ihipGetMemcpyParam3DCommand(command, &desc, queue); -} - -hipError_t ihipMemcpy3D(const hipMemcpy3DParms* p, hipStream_t stream, bool isAsync = false) { - hipError_t status = ihipMemcpy3D_validate(p); - if (status != hipSuccess) { - return status; - } - const HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*p); - - return ihipMemcpyParam3D(&desc, stream, isAsync); -} - -hipError_t hipMemcpy3D(const hipMemcpy3DParms* p) { - HIP_INIT_API(hipMemcpy3D, p); - - HIP_RETURN_DURATION(ihipMemcpy3D(p, nullptr)); -} - -hipError_t hipMemcpy3DAsync(const hipMemcpy3DParms* p, hipStream_t stream) { - HIP_INIT_API(hipMemcpy3DAsync, p, stream); - - STREAM_CAPTURE(hipMemcpy3DAsync, stream, p); - - HIP_RETURN_DURATION(ihipMemcpy3D(p, stream, true)); -} - -hipError_t hipDrvMemcpy3D(const HIP_MEMCPY3D* pCopy) { - HIP_INIT_API(hipDrvMemcpy3D, pCopy); - - HIP_RETURN_DURATION(ihipMemcpyParam3D(pCopy, nullptr)); -} - -hipError_t hipDrvMemcpy3DAsync(const HIP_MEMCPY3D* pCopy, hipStream_t stream) { - HIP_INIT_API(hipDrvMemcpy3DAsync, pCopy, stream); - - HIP_RETURN_DURATION(ihipMemcpyParam3D(pCopy, stream, true)); -} - -hipError_t packFillMemoryCommand(amd::Command*& command, amd::Memory* memory, size_t offset, - int64_t value, size_t valueSize, size_t sizeBytes, - amd::HostQueue* queue) { - if ((memory == nullptr) || (queue == nullptr)) { - return hipErrorInvalidValue; - } - - amd::Command::EventWaitList waitList; - amd::Coord3D fillOffset(offset, 0, 0); - amd::Coord3D fillSize(sizeBytes, 1, 1); - command = - new amd::FillMemoryCommand(*queue, CL_COMMAND_FILL_BUFFER, waitList, *memory->asBuffer(), - &value, valueSize, fillOffset, fillSize); - if (command == nullptr) { - return hipErrorOutOfMemory; - } - return hipSuccess; -} - -hipError_t ihipMemset_validate(void* dst, int64_t value, size_t valueSize, - size_t sizeBytes) { - if (sizeBytes == 0) { - // Skip if nothing needs filling. - return hipSuccess; - } - - if (dst == nullptr) { - return hipErrorInvalidValue; - } - - size_t offset = 0; - amd::Memory* memory = getMemoryObject(dst, offset); - if (memory == nullptr) { - // dst ptr is host ptr hence error - return hipErrorInvalidValue; - } - return hipSuccess; -} - -hipError_t ihipMemsetCommand(std::vector& commands, void* dst, int64_t value, - size_t valueSize, size_t sizeBytes, amd::HostQueue* queue) { - hipError_t hip_error = hipSuccess; - auto aligned_dst = amd::alignUp(reinterpret_cast
(dst), sizeof(uint64_t)); - size_t offset = 0; - amd::Memory* memory = getMemoryObject(dst, offset); - size_t n_head_bytes = 0; - size_t n_tail_bytes = 0; - int64_t value64 = 0; - amd::Command* command; - - if (sizeBytes / sizeof(int64_t) > 0) { - n_head_bytes = static_cast(aligned_dst) - static_cast(dst); - n_tail_bytes = ((sizeBytes - n_head_bytes) % sizeof(int64_t)); - offset = offset + n_head_bytes; - size_t n_bytes = sizeBytes - n_tail_bytes - n_head_bytes; - if (n_bytes > 0) { - if (valueSize == sizeof(int8_t)) { - value = value & 0xff; - value64 = ((value << 56) | (value << 48) | (value << 40) | (value << 32) | (value << 24) | - (value << 16) | (value << 8) | (value)); - } else if (valueSize == sizeof(int16_t)) { - value = value & 0xffff; - value64 = ((value << 48) | (value << 32) | (value << 16) | (value)); - } else if (valueSize == sizeof(int32_t)) { - value = value & 0xffffffff; - value64 = ((value << 32) | (value)); - } else if (valueSize == sizeof(int64_t)) { - value64 = value; - } else { - LogPrintfError("Unsupported Pattern size: %u \n", valueSize); - return hipErrorInvalidValue; - } - // If n_tail_bytes is != 0 then we will do a second fillBuffer Command - // on the same stream below, dont wait, do the first call async. - hip_error = - packFillMemoryCommand(command, memory, offset, value64, sizeof(int64_t), n_bytes, queue); - commands.push_back(command); - } - if (hip_error != hipSuccess) { - return hip_error; - } - } else { - n_head_bytes = sizeBytes; - } - - if (n_head_bytes != 0) { - memory = getMemoryObject(dst, offset); - hip_error = - packFillMemoryCommand(command, memory, offset, value, valueSize, n_head_bytes, queue); - commands.push_back(command); - } - - if (n_tail_bytes != 0) { - void* new_dst = (reinterpret_cast
(dst) + sizeBytes) - n_tail_bytes; - memory = getMemoryObject(new_dst, offset); - hip_error = - packFillMemoryCommand(command, memory, offset, value, valueSize, n_tail_bytes, queue); - commands.push_back(command); - } - return hip_error; -} - -hipError_t ihipMemset(void* dst, int64_t value, size_t valueSize, size_t sizeBytes, - hipStream_t stream, bool isAsync = false) { - hipError_t hip_error = hipSuccess; - hip_error = ihipMemset_validate(dst, value, valueSize, sizeBytes); - if (hip_error != hipSuccess) { - return hip_error; - } - std::vector commands; - amd::HostQueue* queue = hip::getQueue(stream); - hip_error = ihipMemsetCommand(commands, dst, value, valueSize, sizeBytes, queue); - if (hip_error != hipSuccess) { - return hip_error; - } - for (auto command : commands) { - command->enqueue(); - if (!isAsync) { - command->awaitCompletion(); - } - command->release(); - } - return hip_error; -} - -hipError_t hipMemset(void* dst, int value, size_t sizeBytes) { - HIP_INIT_API(hipMemset, dst, value, sizeBytes); - - HIP_RETURN(ihipMemset(dst, value, sizeof(int8_t), sizeBytes, nullptr)); -} - -hipError_t hipMemsetAsync(void* dst, int value, size_t sizeBytes, hipStream_t stream) { - HIP_INIT_API(hipMemsetAsync, dst, value, sizeBytes, stream); - size_t valueSize = sizeof(int8_t); - STREAM_CAPTURE(hipMemsetAsync, stream, dst, value, valueSize, sizeBytes); - - HIP_RETURN(ihipMemset(dst, value, sizeof(int8_t), sizeBytes, stream, true)); -} - -hipError_t hipMemsetD8(hipDeviceptr_t dst, unsigned char value, size_t count) { - HIP_INIT_API(hipMemsetD8, dst, value, count); - - HIP_RETURN(ihipMemset(dst, value, sizeof(int8_t), count * sizeof(int8_t), nullptr)); -} - -hipError_t hipMemsetD8Async(hipDeviceptr_t dst, unsigned char value, size_t count, - hipStream_t stream) { - HIP_INIT_API(hipMemsetD8Async, dst, value, count, stream); - - HIP_RETURN(ihipMemset(dst, value, sizeof(int8_t), count * sizeof(int8_t), stream, true)); -} - -hipError_t hipMemsetD16(hipDeviceptr_t dst, unsigned short value, size_t count) { - HIP_INIT_API(hipMemsetD16, dst, value, count); - - HIP_RETURN(ihipMemset(dst, value, sizeof(int16_t), count * sizeof(int16_t), nullptr)); -} - -hipError_t hipMemsetD16Async(hipDeviceptr_t dst, unsigned short value, size_t count, - hipStream_t stream) { - HIP_INIT_API(hipMemsetD16Async, dst, value, count, stream); - - HIP_RETURN(ihipMemset(dst, value, sizeof(int16_t), count * sizeof(int16_t), stream, true)); -} - -hipError_t hipMemsetD32(hipDeviceptr_t dst, int value, size_t count) { - HIP_INIT_API(hipMemsetD32, dst, value, count); - - HIP_RETURN(ihipMemset(dst, value, sizeof(int32_t), count * sizeof(int32_t), nullptr)); -} - -hipError_t hipMemsetD32Async(hipDeviceptr_t dst, int value, size_t count, - hipStream_t stream) { - HIP_INIT_API(hipMemsetD32Async, dst, value, count, stream); - - HIP_RETURN(ihipMemset(dst, value, sizeof(int32_t), count * sizeof(int32_t), stream, true)); -} - -hipError_t ihipMemset3D_validate(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent, - size_t sizeBytes) { - size_t offset = 0; - amd::Memory* memory = getMemoryObject(pitchedDevPtr.ptr, offset); - - if (memory == nullptr) { - return hipErrorInvalidValue; - } - if (sizeBytes > memory->getSize()) { - return hipErrorInvalidValue; - } - return hipSuccess; -} - -hipError_t ihipMemset3DCommand(std::vector &commands, hipPitchedPtr pitchedDevPtr, - int value, hipExtent extent, amd::HostQueue* queue) { - size_t offset = 0; - auto sizeBytes = extent.width * extent.height * extent.depth; - amd::Memory* memory = getMemoryObject(pitchedDevPtr.ptr, offset); - if (pitchedDevPtr.pitch == extent.width) { - return ihipMemsetCommand(commands, pitchedDevPtr.ptr, value, sizeof(int8_t), - static_cast(sizeBytes), queue); - } - // Workaround for cases when pitch > row until fill kernel will be updated to support pitch. - // Fall back to filling one row at a time. - amd::Coord3D origin(offset); - amd::Coord3D region(pitchedDevPtr.xsize, pitchedDevPtr.ysize, extent.depth); - amd::BufferRect rect; - if (pitchedDevPtr.pitch == 0 || - !rect.create(static_cast(origin), static_cast(region), pitchedDevPtr.pitch, - 0)) { - return hipErrorInvalidValue; - } - amd::FillMemoryCommand* command; - for (size_t slice = 0; slice < extent.depth; slice++) { - for (size_t row = 0; row < extent.height; row++) { - const size_t rowOffset = rect.offset(0, row, slice); - command = new amd::FillMemoryCommand( - *queue, CL_COMMAND_FILL_BUFFER, amd::Command::EventWaitList{}, *memory->asBuffer(), - &value, sizeof(int8_t), amd::Coord3D{rowOffset, 0, 0}, amd::Coord3D{extent.width, 1, 1}); - commands.push_back(command); - } - } - return hipSuccess; -} - - -hipError_t ihipMemset3D(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent, - hipStream_t stream, bool isAsync = false) { - auto sizeBytes = extent.width * extent.height * extent.depth; - - if (sizeBytes == 0) { - // sizeBytes is zero hence returning early as nothing to be set - return hipSuccess; - } - hipError_t status = ihipMemset3D_validate(pitchedDevPtr, value, extent, sizeBytes); - if (status != hipSuccess) { - return status; - } - amd::HostQueue* queue = hip::getQueue(stream); - std::vector commands; - ihipMemset3DCommand(commands, pitchedDevPtr, value, extent, queue); - for (auto& command : commands) { - command->enqueue(); - if (!isAsync) { - command->awaitCompletion(); - } - command->release(); - } - return hipSuccess; -} - -hipError_t hipMemset2D(void* dst, size_t pitch, int value, size_t width, size_t height) { - HIP_INIT_API(hipMemset2D, dst, pitch, value, width, height); - - HIP_RETURN(ihipMemset3D({dst, pitch, width, height}, value, {width, height, 1}, nullptr)); -} - -hipError_t hipMemset2DAsync(void* dst, size_t pitch, int value, - size_t width, size_t height, hipStream_t stream) { - HIP_INIT_API(hipMemset2DAsync, dst, pitch, value, width, height, stream); - - STREAM_CAPTURE(hipMemset2DAsync, stream, dst, pitch, value, width, height); - - HIP_RETURN(ihipMemset3D({dst, pitch, width, height}, value, {width, height, 1}, stream, true)); -} - -hipError_t hipMemset3D(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent) { - HIP_INIT_API(hipMemset3D, pitchedDevPtr, value, extent); - - HIP_RETURN(ihipMemset3D(pitchedDevPtr, value, extent, nullptr)); -} - -hipError_t hipMemset3DAsync(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent, hipStream_t stream) { - HIP_INIT_API(hipMemset3DAsync, pitchedDevPtr, value, extent, stream); - - STREAM_CAPTURE(hipMemset3DAsync, stream, pitchedDevPtr, value, extent); - - HIP_RETURN(ihipMemset3D(pitchedDevPtr, value, extent, stream, true)); -} - -hipError_t hipMemAllocPitch(hipDeviceptr_t* dptr, size_t* pitch, size_t widthInBytes, - size_t height, unsigned int elementSizeBytes) { - HIP_INIT_API(hipMemAllocPitch, dptr, pitch, widthInBytes, height, elementSizeBytes); - - HIP_RETURN(hipMallocPitch(dptr, pitch, widthInBytes, height)); -} - -hipError_t hipMemAllocHost(void** ptr, size_t size) { - HIP_INIT_API(hipMemAllocHost, ptr, size); - - HIP_RETURN_DURATION(hipHostMalloc(ptr, size, 0)); -} - -hipError_t hipIpcGetMemHandle(hipIpcMemHandle_t* handle, void* dev_ptr) { - HIP_INIT_API(hipIpcGetMemHandle, handle, dev_ptr); - - amd::Device* device = nullptr; - ihipIpcMemHandle_t* ihandle = nullptr; - - if ((handle == nullptr) || (dev_ptr == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - device = hip::getCurrentDevice()->devices()[0]; - ihandle = reinterpret_cast(handle); - - if(!device->IpcCreate(dev_ptr, &(ihandle->psize), &(ihandle->ipc_handle), &(ihandle->poffset))) { - LogPrintfError("IPC memory creation failed for memory: 0x%x", dev_ptr); - HIP_RETURN(hipErrorInvalidDevicePointer); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipIpcOpenMemHandle(void** dev_ptr, hipIpcMemHandle_t handle, unsigned int flags) { - HIP_INIT_API(hipIpcOpenMemHandle, dev_ptr, &handle, flags); - - amd::Memory* amd_mem_obj = nullptr; - amd::Device* device = nullptr; - ihipIpcMemHandle_t* ihandle = nullptr; - - if (dev_ptr == nullptr || flags != hipIpcMemLazyEnablePeerAccess) { - HIP_RETURN(hipErrorInvalidValue); - } - - /* Call the IPC Attach from Device class */ - device = hip::getCurrentDevice()->devices()[0]; - ihandle = reinterpret_cast(&handle); - - if (ihandle->psize == 0) { - HIP_RETURN(hipErrorInvalidValue); - } - - if(!device->IpcAttach(&(ihandle->ipc_handle), ihandle->psize, - ihandle->poffset, flags, dev_ptr)) { - LogPrintfError("Cannot attach ipc_handle: with ipc_size: %u" - "ipc_offset: %u flags: %u", ihandle->psize, flags); - HIP_RETURN(hipErrorInvalidDevicePointer); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipIpcCloseMemHandle(void* dev_ptr) { - HIP_INIT_API(hipIpcCloseMemHandle, dev_ptr); - - size_t offset = 0; - amd::Device* device = nullptr; - amd::Memory* amd_mem_obj = nullptr; - - hip::getNullStream()->finish(); - - if (dev_ptr == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - /* Call IPC Detach from Device class */ - device = hip::getCurrentDevice()->devices()[0]; - if (device == nullptr) { - HIP_RETURN(hipErrorNoDevice); - } - - /* detach the memory */ - if (!device->IpcDetach(dev_ptr)){ - HIP_RETURN(hipErrorInvalidHandle); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipHostGetDevicePointer(void** devicePointer, void* hostPointer, unsigned flags) { - HIP_INIT_API(hipHostGetDevicePointer, devicePointer, hostPointer, flags); - - size_t offset = 0; - - amd::Memory* memObj = getMemoryObject(hostPointer, offset); - if (!memObj) { - HIP_RETURN(hipErrorInvalidValue); - } -*devicePointer = reinterpret_cast(memObj->getDeviceMemory(*hip::getCurrentDevice()->devices()[0])->virtualAddress() + offset); - - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipPointerGetAttributes(hipPointerAttribute_t* attributes, const void* ptr) { - HIP_INIT_API(hipPointerGetAttributes, attributes, ptr); - - if (attributes == nullptr || ptr == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - size_t offset = 0; - amd::Memory* memObj = getMemoryObject(ptr, offset); - int device = 0; - memset(attributes, 0, sizeof(hipPointerAttribute_t)); - - if (memObj != nullptr) { - attributes->memoryType = ((CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_USE_HOST_PTR) & - memObj->getMemFlags())? hipMemoryTypeHost : hipMemoryTypeDevice; - if (attributes->memoryType == hipMemoryTypeHost) { - if (memObj->getHostMem() != nullptr) { - attributes->hostPointer = static_cast(memObj->getHostMem()) + offset; - } - else { - attributes->hostPointer = static_cast(memObj->getSvmPtr()) + offset; - } - } - - device::Memory* devMem = memObj->getDeviceMemory(*hip::getCurrentDevice()->devices()[0]); - //getDeviceMemory can fail, hence validate the sanity of the mem obtained - if (nullptr == devMem) { - DevLogPrintfError("getDeviceMemory for ptr failed : %p \n", ptr); - HIP_RETURN(hipErrorMemoryAllocation); - } - - attributes->devicePointer = reinterpret_cast(devMem->virtualAddress() + offset); - constexpr uint32_t kManagedAlloc = (CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_ALLOC_HOST_PTR); - attributes->isManaged = - ((memObj->getMemFlags() & kManagedAlloc) == kManagedAlloc) ? true : false; - attributes->allocationFlags = memObj->getMemFlags() >> 16; - - amd::Context* memObjCtx = &memObj->getContext(); - if (hip::host_device->asContext() == memObjCtx) { - attributes->device = ihipGetDevice(); - HIP_RETURN(hipSuccess); - } - for (auto& ctx : g_devices) { - if (ctx->asContext() == memObjCtx) { - attributes->device = device; - HIP_RETURN(hipSuccess); - } - ++device; - } - LogPrintfError("Cannot find memory object context, memObjCtx: 0x%x \n", memObjCtx); - HIP_RETURN(hipErrorInvalidDevice); - } - - LogPrintfError("Cannot get amd_mem_obj for ptr: 0x%x \n", ptr); - HIP_RETURN(hipErrorInvalidValue); -} - -// ================================================================================================ -hipError_t hipArrayDestroy(hipArray* array) { - HIP_INIT_API(hipArrayDestroy, array); - - HIP_RETURN(ihipArrayDestroy(array)); -} - -hipError_t hipArray3DGetDescriptor(HIP_ARRAY3D_DESCRIPTOR* pArrayDescriptor, - hipArray* array) { - HIP_INIT_API(hipArray3DGetDescriptor, pArrayDescriptor, array); - - assert(false && "Unimplemented"); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipArrayGetDescriptor(HIP_ARRAY_DESCRIPTOR* pArrayDescriptor, - hipArray* array) { - HIP_INIT_API(hipArrayGetDescriptor, pArrayDescriptor, array); - - assert(false && "Unimplemented"); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipMemcpyParam2DAsync(const hip_Memcpy2D* pCopy, - hipStream_t stream) { - HIP_INIT_API(hipMemcpyParam2DAsync, pCopy); - - HIP_RETURN(ihipMemcpyParam2D(pCopy, stream, true)); -} - -hipError_t ihipMemcpy2DArrayToArray(hipArray_t dst, size_t wOffsetDst, size_t hOffsetDst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { - hip_Memcpy2D desc = {}; - - desc.srcXInBytes = wOffsetSrc; - desc.srcY = hOffsetSrc; - desc.srcMemoryType = hipMemoryTypeArray; - desc.srcHost = nullptr; - desc.srcDevice = nullptr; - desc.srcArray = const_cast(src); - desc.srcPitch = 0; // Ignored. - - desc.dstXInBytes = wOffsetDst; - desc.dstY = hOffsetDst; - desc.dstMemoryType = hipMemoryTypeArray; - desc.dstHost = nullptr; - desc.dstDevice = nullptr; - desc.dstArray = dst; - desc.dstPitch = 0; // Ignored. - - desc.WidthInBytes = width; - desc.Height = height; - - return ihipMemcpyParam2D(&desc, stream, isAsync); -} - -hipError_t hipMemcpy2DArrayToArray(hipArray_t dst, size_t wOffsetDst, size_t hOffsetDst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpy2DArrayToArray, dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind); - - HIP_RETURN_DURATION(ihipMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind, nullptr)); -} - -hipError_t hipMemcpyArrayToArray(hipArray_t dst, size_t wOffsetDst, size_t hOffsetDst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyArrayToArray, dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind); - - HIP_RETURN_DURATION(ihipMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind, nullptr)); -} - -hipError_t hipMemcpy2DFromArray(void* dst, size_t dpitch, hipArray_const_t src, size_t wOffsetSrc, size_t hOffset, size_t width, size_t height, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpy2DFromArray, dst, dpitch, src, wOffsetSrc, hOffset, width, height, kind); - - if (dpitch == 0) { - HIP_RETURN(hipErrorInvalidPitchValue); - } - - HIP_RETURN_DURATION(ihipMemcpy2DFromArray(dst, dpitch, src, wOffsetSrc, hOffset, width, height, kind, nullptr)); -} - -hipError_t hipMemcpy2DFromArrayAsync(void* dst, size_t dpitch, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpy2DFromArrayAsync, dst, dpitch, src, wOffsetSrc, hOffsetSrc, width, height, kind, stream); - - if (dpitch == 0) { - HIP_RETURN(hipErrorInvalidPitchValue); - } - - HIP_RETURN_DURATION(ihipMemcpy2DFromArray(dst, dpitch, src, wOffsetSrc, hOffsetSrc, width, height, kind, stream, true)); -} - -hipError_t hipMemcpyFromArrayAsync(void* dst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpyFromArrayAsync, dst, src, wOffsetSrc, hOffsetSrc, count, kind, stream); - - if (src == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - const size_t arrayHeight = (src->height != 0) ? src->height : 1; - const size_t widthInBytes = count / arrayHeight; - - const size_t height = (count / src->width) / hip::getElementSize(src); - - HIP_RETURN_DURATION(ihipMemcpy2DFromArray(dst, 0 /* dpitch */, src, wOffsetSrc, hOffsetSrc, widthInBytes, height, kind, stream, true)); -} - -hipError_t hipMemcpy2DToArrayAsync(hipArray* dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpy2DToArrayAsync, dst, wOffset, hOffset, src, spitch, width, height, kind); - - if (spitch == 0) { - HIP_RETURN(hipErrorInvalidPitchValue); - } - - HIP_RETURN_DURATION(ihipMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind, stream, true)); -} - -hipError_t hipMemcpyToArrayAsync(hipArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpyToArrayAsync, dst, wOffset, hOffset, src, count, kind); - - if (dst == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - const size_t arrayHeight = (dst->height != 0) ? dst->height : 1; - const size_t widthInBytes = count / arrayHeight; - - const size_t height = (count / dst->width) / hip::getElementSize(dst); - - HIP_RETURN_DURATION(ihipMemcpy2DToArray(dst, wOffset, hOffset, src, 0 /* spitch */, widthInBytes, height, kind, stream, true)); -} - -hipError_t hipMemcpyAtoA(hipArray* dstArray, - size_t dstOffset, - hipArray* srcArray, - size_t srcOffset, - size_t ByteCount) { - HIP_INIT_API(hipMemcpyAtoA, dstArray, dstOffset, srcArray, srcOffset, ByteCount); - - HIP_RETURN_DURATION(ihipMemcpyAtoA(srcArray, dstArray, {srcOffset, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, nullptr)); -} - -hipError_t hipMemcpyAtoD(hipDeviceptr_t dstDevice, - hipArray* srcArray, - size_t srcOffset, - size_t ByteCount) { - HIP_INIT_API(hipMemcpyAtoD, dstDevice, srcArray, srcOffset, ByteCount); - - HIP_RETURN_DURATION(ihipMemcpyAtoD(srcArray, dstDevice, {srcOffset, 0, 0}, {0, 0, 0}, {ByteCount, 1, 1}, 0, 0, nullptr)); -} - -hipError_t hipMemcpyAtoHAsync(void* dstHost, - hipArray* srcArray, - size_t srcOffset, - size_t ByteCount, - hipStream_t stream) { - HIP_INIT_API(hipMemcpyAtoHAsync, dstHost, srcArray, srcOffset, ByteCount, stream); - - HIP_RETURN_DURATION(ihipMemcpyAtoH(srcArray, dstHost, {srcOffset, 0, 0}, {0, 0, 0}, {ByteCount, 1, 1}, 0, 0, stream, true)); -} - -hipError_t hipMemcpyDtoA(hipArray* dstArray, - size_t dstOffset, - hipDeviceptr_t srcDevice, - size_t ByteCount) { - HIP_INIT_API(hipMemcpyDtoA, dstArray, dstOffset, srcDevice, ByteCount); - - HIP_RETURN_DURATION(ihipMemcpyDtoA(srcDevice, dstArray, {0, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, 0, 0, nullptr)); -} - -hipError_t hipMemcpyHtoAAsync(hipArray* dstArray, - size_t dstOffset, - const void* srcHost, - size_t ByteCount, - hipStream_t stream) { - HIP_INIT_API(hipMemcpyHtoAAsync, dstArray, dstOffset, srcHost, ByteCount, stream); - - HIP_RETURN_DURATION(ihipMemcpyHtoA(srcHost, dstArray, {0, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, 0, 0, stream, true)); -} - -hipError_t hipMallocHost(void** ptr, - size_t size) { - HIP_INIT_API(hipMallocHost, ptr, size); - - HIP_RETURN_DURATION(ihipMalloc(ptr, size, CL_MEM_SVM_FINE_GRAIN_BUFFER), (ptr != nullptr)? *ptr : nullptr); -} - -hipError_t hipFreeHost(void *ptr) { - HIP_INIT_API(hipFreeHost, ptr); - - HIP_RETURN(ihipFree(ptr)); -} - -hipError_t hipDrvMemcpy2DUnaligned(const hip_Memcpy2D* pCopy) { - HIP_INIT_API(hipDrvMemcpy2DUnaligned, pCopy); - - HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*pCopy); - - HIP_RETURN(ihipMemcpyParam3D(&desc, nullptr)); -} - -hipError_t hipMallocMipmappedArray(hipMipmappedArray_t *mipmappedArray, - const hipChannelFormatDesc* desc, - hipExtent extent, - unsigned int numLevels, - unsigned int flags) { - HIP_INIT_API(hipMallocMipmappedArray, mipmappedArray, desc, extent, numLevels, flags); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipFreeMipmappedArray(hipMipmappedArray_t mipmappedArray) { - HIP_INIT_API(hipFreeMipmappedArray, mipmappedArray); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipGetMipmappedArrayLevel(hipArray_t *levelArray, - hipMipmappedArray_const_t mipmappedArray, - unsigned int level) { - HIP_INIT_API(hipGetMipmappedArrayLevel, levelArray, mipmappedArray, level); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t ihipMipmapArrayCreate(hipMipmappedArray_t* mipmapped_array_pptr, - HIP_ARRAY3D_DESCRIPTOR* mipmapped_array_desc_ptr, - unsigned int num_mipmap_levels) { - - const cl_channel_order channel_order = hip::getCLChannelOrder( - mipmapped_array_desc_ptr->NumChannels, 0); - const cl_channel_type channel_type = hip::getCLChannelType(mipmapped_array_desc_ptr->Format, - hipReadModeElementType); - const cl_mem_object_type image_type = hip::getCLMemObjectType(mipmapped_array_desc_ptr->Width, - mipmapped_array_desc_ptr->Height, - mipmapped_array_desc_ptr->Depth, - mipmapped_array_desc_ptr->Flags); - - // Create a new amd::Image with mipmap - amd::Image* image = ihipImageCreate(channel_order, - channel_type, - image_type, - mipmapped_array_desc_ptr->Width, - mipmapped_array_desc_ptr->Height, - mipmapped_array_desc_ptr->Depth, - mipmapped_array_desc_ptr->Depth, - 0 /* row pitch */, - 0 /* slice pitch */, - num_mipmap_levels, - nullptr /* buffer */); - - if (image == nullptr) { - return hipErrorInvalidValue; - } - - cl_mem cl_mem_obj = as_cl(image); - *mipmapped_array_pptr = new hipMipmappedArray(); - (*mipmapped_array_pptr)->data = reinterpret_cast(cl_mem_obj); - - (*mipmapped_array_pptr)->desc = hip::getChannelFormatDesc( - mipmapped_array_desc_ptr->NumChannels, - mipmapped_array_desc_ptr->Format); - (*mipmapped_array_pptr)->type = image_type; - (*mipmapped_array_pptr)->width = mipmapped_array_desc_ptr->Width; - (*mipmapped_array_pptr)->height = mipmapped_array_desc_ptr->Height; - (*mipmapped_array_pptr)->depth = mipmapped_array_desc_ptr->Depth; - (*mipmapped_array_pptr)->min_mipmap_level = 0; - (*mipmapped_array_pptr)->max_mipmap_level = num_mipmap_levels; - (*mipmapped_array_pptr)->flags = mipmapped_array_desc_ptr->Flags; - (*mipmapped_array_pptr)->format = mipmapped_array_desc_ptr->Format; - - return hipSuccess; -} - -hipError_t ihipMipmappedArrayDestroy(hipMipmappedArray_t mipmapped_array_ptr) { - - if (mipmapped_array_ptr == nullptr) { - return hipErrorInvalidValue; - } - - cl_mem mem_obj = reinterpret_cast(mipmapped_array_ptr->data); - if (is_valid(mem_obj) == false) { - return hipErrorInvalidValue; - } - - for (auto& dev : g_devices) { - dev->NullStream()->finish(); - } - - as_amd(mem_obj)->release(); - - delete mipmapped_array_ptr; - - return hipSuccess; -} - -hipError_t ihipMipmappedArrayGetLevel(hipArray_t* level_array_pptr, - hipMipmappedArray_t mipmapped_array_ptr, - unsigned int mip_level) { - - if (level_array_pptr == nullptr || mipmapped_array_ptr == nullptr) { - return hipErrorInvalidValue; - } - - // Convert the raw data to amd::Image - cl_mem cl_mem_obj = reinterpret_cast(mipmapped_array_ptr->data); - if (is_valid(cl_mem_obj) == false) { - return hipErrorInvalidValue; - } - - amd::Image* image = as_amd(cl_mem_obj)->asImage(); - if (image == nullptr) { - return hipErrorInvalidValue; - } - - // Create new hip Array parameter and create an image view with new mip level. - (*level_array_pptr) = new hipArray(); - (*level_array_pptr)->data = as_cl(image->createView(image->getContext(), - image->getImageFormat(), - NULL, mip_level, 0)); - - // Copy the new width, height & depth details of the flag to hipArray. - cl_mem cl_mip_mem_obj = reinterpret_cast((*level_array_pptr)->data); - if (is_valid(cl_mem_obj) == false) { - return hipErrorInvalidValue; - } - - // Fill the hip_array info from newly created amd::Image's view - amd::Image* mipmap_image = as_amd(cl_mip_mem_obj)->asImage(); - (*level_array_pptr)->width = mipmap_image->getWidth(); - (*level_array_pptr)->height = mipmap_image->getHeight(); - (*level_array_pptr)->depth = mipmap_image->getDepth(); - - const cl_mem_object_type image_type = hip::getCLMemObjectType((*level_array_pptr)->width, - (*level_array_pptr)->height, - (*level_array_pptr)->depth, - mipmapped_array_ptr->flags); - (*level_array_pptr)->type = image_type; - (*level_array_pptr)->Format = mipmapped_array_ptr->format; - (*level_array_pptr)->desc = mipmapped_array_ptr->desc; - (*level_array_pptr)->NumChannels = hip::getNumChannels((*level_array_pptr)->desc); - (*level_array_pptr)->isDrv = 0; - (*level_array_pptr)->textureType = 0; - - return hipSuccess; -} - -hipError_t hipMipmappedArrayCreate(hipMipmappedArray_t* mipmapped_array_pptr, - HIP_ARRAY3D_DESCRIPTOR* mipmapped_array_desc_ptr, - unsigned int num_mipmap_levels) { - HIP_INIT_API(hipMipmappedArrayCreate, mipmapped_array_pptr, mipmapped_array_desc_ptr, - num_mipmap_levels); - - HIP_RETURN(ihipMipmapArrayCreate(mipmapped_array_pptr, mipmapped_array_desc_ptr, - num_mipmap_levels)); -} - -hipError_t hipMipmappedArrayDestroy(hipMipmappedArray_t mipmapped_array_ptr) { - HIP_INIT_API(hipMipmappedArrayDestroy, mipmapped_array_ptr); - - HIP_RETURN(ihipMipmappedArrayDestroy(mipmapped_array_ptr)); -} - -hipError_t hipMipmappedArrayGetLevel(hipArray_t* level_array_pptr, - hipMipmappedArray_t mipmapped_array_ptr, - unsigned int mip_level) { - HIP_INIT_API(hipMipmappedArrayGetLevel, level_array_pptr, mipmapped_array_ptr, mip_level); - - HIP_RETURN(ihipMipmappedArrayGetLevel(level_array_pptr, mipmapped_array_ptr, mip_level)); -} - diff --git a/src/hipamd/src/hip_module.cpp b/src/hipamd/src/hip_module.cpp deleted file mode 100755 index 59401dbdd2..0000000000 --- a/src/hipamd/src/hip_module.cpp +++ /dev/null @@ -1,675 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include -#include - -#include "hip_internal.hpp" -#include "platform/program.hpp" -#include "hip_event.hpp" -#include "hip_platform.hpp" - -hipError_t ihipModuleLoadData(hipModule_t* module, const void* mmap_ptr, size_t mmap_size); - -extern hipError_t ihipLaunchKernel(const void* hostFunction, - dim3 gridDim, - dim3 blockDim, - void** args, - size_t sharedMemBytes, - hipStream_t stream, - hipEvent_t startEvent, - hipEvent_t stopEvent, - int flags); - -const std::string& FunctionName(const hipFunction_t f) { - return hip::DeviceFunc::asFunction(f)->kernel()->name(); -} - -static uint64_t ElfSize(const void *emi) -{ - return amd::Elf::getElfSize(emi); -} - -hipError_t hipModuleUnload(hipModule_t hmod) { - HIP_INIT_API(hipModuleUnload, hmod); - - HIP_RETURN(PlatformState::instance().unloadModule(hmod)); -} - -hipError_t hipModuleLoad(hipModule_t* module, const char* fname) { - HIP_INIT_API(hipModuleLoad, module, fname); - - HIP_RETURN(PlatformState::instance().loadModule(module, fname)); -} - -hipError_t hipModuleLoadData(hipModule_t *module, const void *image) -{ - HIP_INIT_API(hipModuleLoadData, module, image); - - HIP_RETURN(PlatformState::instance().loadModule(module, 0, image)); -} - -hipError_t hipModuleLoadDataEx(hipModule_t *module, const void *image, - unsigned int numOptions, hipJitOption* options, - void** optionsValues) -{ - /* TODO: Pass options to Program */ - HIP_INIT_API(hipModuleLoadDataEx, module, image); - - HIP_RETURN(PlatformState::instance().loadModule(module, 0, image)); -} - -extern hipError_t __hipExtractCodeObjectFromFatBinary(const void* data, - const std::vector& devices, - std::vector>& code_objs); - -hipError_t hipModuleGetFunction(hipFunction_t *hfunc, hipModule_t hmod, const char *name) { - HIP_INIT_API(hipModuleGetFunction, hfunc, hmod, name); - - if(hfunc == nullptr || name == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - if (hipSuccess != PlatformState::instance().getDynFunc(hfunc, hmod, name)) { - LogPrintfError("Cannot find the function: %s for module: 0x%x \n", name, hmod); - HIP_RETURN(hipErrorNotFound); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes, hipModule_t hmod, const char* name) -{ - HIP_INIT_API(hipModuleGetGlobal, dptr, bytes, hmod, name); - - if(dptr == nullptr || bytes == nullptr || name == nullptr) { - return hipErrorInvalidValue; - } - - /* Get address and size for the global symbol */ - if (hipSuccess != PlatformState::instance().getDynGlobalVar(name, hmod, dptr, bytes)) { - LogPrintfError("Cannot find global Var: %s for module: 0x%x at device: %d \n", name, hmod, - ihipGetDevice()); - HIP_RETURN(hipErrorNotFound); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipFuncGetAttribute(int* value, hipFunction_attribute attrib, hipFunction_t hfunc) { - HIP_INIT_API(hipFuncGetAttribute, value, attrib, hfunc); - - if ((value == nullptr) || (hfunc == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - hip::DeviceFunc* function = hip::DeviceFunc::asFunction(hfunc); - if (function == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - - amd::Kernel* kernel = function->kernel(); - if (kernel == nullptr) { - HIP_RETURN(hipErrorInvalidDeviceFunction); - } - - const device::Kernel::WorkGroupInfo* wrkGrpInfo - = kernel->getDeviceKernel(*(hip::getCurrentDevice()->devices()[0]))->workGroupInfo(); - if (wrkGrpInfo == nullptr) { - HIP_RETURN(hipErrorMissingConfiguration); - } - - switch(attrib) { - case HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES: - *value = static_cast(wrkGrpInfo->localMemSize_); - break; - case HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK: - *value = static_cast(wrkGrpInfo->size_); - break; - case HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES: - *value = 0; - break; - case HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES: - *value = static_cast(wrkGrpInfo->privateMemSize_); - break; - case HIP_FUNC_ATTRIBUTE_NUM_REGS: - *value = static_cast(wrkGrpInfo->usedVGPRs_); - break; - case HIP_FUNC_ATTRIBUTE_PTX_VERSION: - *value = 30; // Defaults to 3.0 as HCC - break; - case HIP_FUNC_ATTRIBUTE_BINARY_VERSION: - *value = static_cast(kernel->signature().version()); - break; - case HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA: - *value = 0; - break; - case HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES: - *value = static_cast(wrkGrpInfo->availableLDSSize_ - wrkGrpInfo->localMemSize_); - break; - case HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT: - *value = 0; - break; - default: - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipFuncGetAttributes(hipFuncAttributes* attr, const void* func) -{ - HIP_INIT_API(hipFuncGetAttributes, attr, func); - - HIP_RETURN_ONFAIL(PlatformState::instance().getStatFuncAttr(attr, func, ihipGetDevice())); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipFuncSetAttribute ( const void* func, hipFuncAttribute attr, int value ) { - HIP_INIT_API(hipFuncSetAttribute, func, attr, value); - - // No way to set function attribute yet. - - HIP_RETURN(hipSuccess); -} - -hipError_t hipFuncSetCacheConfig (const void* func, hipFuncCache_t cacheConfig) { - - HIP_INIT_API(hipFuncSetCacheConfig, cacheConfig); - - // No way to set cache config yet. - - HIP_RETURN(hipSuccess); -} - -hipError_t hipFuncSetSharedMemConfig ( const void* func, hipSharedMemConfig config) { - HIP_INIT_API(hipFuncSetSharedMemConfig, func, config); - - // No way to set Shared Memory config function yet. - - HIP_RETURN(hipSuccess); -} - -hipError_t ihipLaunchKernel_validate(hipFunction_t f, uint32_t globalWorkSizeX, - uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, - uint32_t blockDimX, uint32_t blockDimY, - uint32_t blockDimZ, uint32_t sharedMemBytes, - void** kernelParams, void** extra, int deviceId, - uint32_t params = 0) { - if (f == nullptr) { - LogPrintfError("%s", "Function passed is null"); - return hipErrorInvalidImage; - } - if ((kernelParams != nullptr) && (extra != nullptr)) { - LogPrintfError("%s", - "Both, kernelParams and extra Params are provided, only one should be provided"); - return hipErrorInvalidValue; - } - if (globalWorkSizeX == 0 || globalWorkSizeY == 0 || globalWorkSizeZ == 0 || blockDimX == 0 || - blockDimY == 0 || blockDimZ == 0) { - return hipErrorInvalidValue; - } - - if (extra != nullptr) { - if (extra[0] != HIP_LAUNCH_PARAM_BUFFER_POINTER || extra[2] != HIP_LAUNCH_PARAM_BUFFER_SIZE || - extra[4] != HIP_LAUNCH_PARAM_END) { - return hipErrorNotInitialized; - } - } - - const amd::Device* device = g_devices[deviceId]->devices()[0]; - // Make sure dispatch doesn't exceed max workgroup size limit - if (blockDimX * blockDimY * blockDimZ > device->info().maxWorkGroupSize_) { - return hipErrorInvalidConfiguration; - } - hip::DeviceFunc* function = hip::DeviceFunc::asFunction(f); - amd::Kernel* kernel = function->kernel(); - // Make sure the launch params are not larger than if specified launch_bounds - // If it exceeds, then return a failure - if (blockDimX * blockDimY * blockDimZ > - kernel->getDeviceKernel(*device)->workGroupInfo()->size_) { - LogPrintfError("Launch params (%u, %u, %u) are larger than launch bounds (%lu) for kernel %s", - blockDimX, blockDimY, blockDimZ, - kernel->getDeviceKernel(*device)->workGroupInfo()->size_, - function->name().c_str()); - return hipErrorLaunchFailure; - } - - if (params & amd::NDRangeKernelCommand::CooperativeGroups) { - if (!device->info().cooperativeGroups_) { - return hipErrorLaunchFailure; - } - int num_blocks = 0; - int max_blocks_per_grid = 0; - int best_block_size = 0; - int block_size = blockDimX * blockDimY * blockDimZ; - hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, &max_blocks_per_grid, - &best_block_size, *device, f, - block_size, sharedMemBytes, true); - if (((globalWorkSizeX * globalWorkSizeY * globalWorkSizeZ) / block_size) > - unsigned(max_blocks_per_grid)) { - return hipErrorCooperativeLaunchTooLarge; - } - } - if (params & amd::NDRangeKernelCommand::CooperativeMultiDeviceGroups) { - if (!device->info().cooperativeMultiDeviceGroups_) { - return hipErrorLaunchFailure; - } - } - address kernargs = nullptr; - // 'extra' is a struct that contains the following info: { - // HIP_LAUNCH_PARAM_BUFFER_POINTER, kernargs, - // HIP_LAUNCH_PARAM_BUFFER_SIZE, &kernargs_size, - // HIP_LAUNCH_PARAM_END } - if (extra != nullptr) { - if (extra[0] != HIP_LAUNCH_PARAM_BUFFER_POINTER || extra[2] != HIP_LAUNCH_PARAM_BUFFER_SIZE || - extra[4] != HIP_LAUNCH_PARAM_END) { - return hipErrorNotInitialized; - } - kernargs = reinterpret_cast
(extra[1]); - } - - const amd::KernelSignature& signature = kernel->signature(); - for (size_t i = 0; i < signature.numParameters(); ++i) { - const amd::KernelParameterDescriptor& desc = signature.at(i); - if (kernelParams == nullptr) { - assert(kernargs != nullptr); - kernel->parameters().set(i, desc.size_, kernargs + desc.offset_, - desc.type_ == T_POINTER /*svmBound*/); - } else { - assert(extra == nullptr); - kernel->parameters().set(i, desc.size_, kernelParams[i], - desc.type_ == T_POINTER /*svmBound*/); - } - } - return hipSuccess; -} - -hipError_t ihipLaunchKernelCommand(amd::Command*& command, hipFunction_t f, - uint32_t globalWorkSizeX, uint32_t globalWorkSizeY, - uint32_t globalWorkSizeZ, uint32_t blockDimX, uint32_t blockDimY, - uint32_t blockDimZ, uint32_t sharedMemBytes, - amd::HostQueue* queue, void** kernelParams, void** extra, - hipEvent_t startEvent = nullptr, hipEvent_t stopEvent = nullptr, - uint32_t flags = 0, uint32_t params = 0, uint32_t gridId = 0, - uint32_t numGrids = 0, uint64_t prevGridSum = 0, - uint64_t allGridSum = 0, uint32_t firstDevice = 0) { - hip::DeviceFunc* function = hip::DeviceFunc::asFunction(f); - amd::Kernel* kernel = function->kernel(); - - size_t globalWorkOffset[3] = {0}; - size_t globalWorkSize[3] = {globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ}; - size_t localWorkSize[3] = {blockDimX, blockDimY, blockDimZ}; - amd::NDRangeContainer ndrange(3, globalWorkOffset, globalWorkSize, localWorkSize); - amd::Command::EventWaitList waitList; - bool profileNDRange = false; - address kernargs = nullptr; - - profileNDRange = (startEvent != nullptr || stopEvent != nullptr); - - // Flag set to 1 signifies that kernel can be launched in anyorder - if (flags & hipExtAnyOrderLaunch) { - params |= amd::NDRangeKernelCommand::AnyOrderLaunch; - } - - amd::NDRangeKernelCommand* kernelCommand = new amd::NDRangeKernelCommand( - *queue, waitList, *kernel, ndrange, sharedMemBytes, params, gridId, numGrids, prevGridSum, - allGridSum, firstDevice, profileNDRange); - if (!kernelCommand) { - return hipErrorOutOfMemory; - } - - // Capture the kernel arguments - if (CL_SUCCESS != kernelCommand->captureAndValidate()) { - delete kernelCommand; - return hipErrorOutOfMemory; - } - command = kernelCommand; - return hipSuccess; -} - -hipError_t ihipModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX, - uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, - uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ, - uint32_t sharedMemBytes, hipStream_t hStream, void** kernelParams, - void** extra, hipEvent_t startEvent, hipEvent_t stopEvent, - uint32_t flags = 0, uint32_t params = 0, uint32_t gridId = 0, - uint32_t numGrids = 0, uint64_t prevGridSum = 0, - uint64_t allGridSum = 0, uint32_t firstDevice = 0) { - HIP_INIT_API(ihipModuleLaunchKernel, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, - blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra, - startEvent, stopEvent, flags, params); - - int deviceId = hip::Stream::DeviceId(hStream); - HIP_RETURN_ONFAIL(PlatformState::instance().initStatManagedVarDevicePtr(deviceId)); - if (f == nullptr) { - LogPrintfError("%s", "Function passed is null"); - return hipErrorInvalidImage; - } - hip::DeviceFunc* function = hip::DeviceFunc::asFunction(f); - amd::Kernel* kernel = function->kernel(); - amd::ScopedLock lock(function->dflock_); - - hipError_t status = - ihipLaunchKernel_validate(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, blockDimX, - blockDimY, blockDimZ, sharedMemBytes, kernelParams, extra, deviceId, params); - if (status != hipSuccess) { - return status; - } - amd::Command* command = nullptr; - amd::HostQueue* queue = hip::getQueue(hStream); - status = ihipLaunchKernelCommand(command, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, - blockDimX, blockDimY, blockDimZ, sharedMemBytes, queue, - kernelParams, extra, startEvent, stopEvent, flags, params, - gridId, numGrids, prevGridSum, allGridSum, firstDevice); - if (status != hipSuccess) { - return status; - } - - hip::Event* eStart = reinterpret_cast(startEvent); - hip::Event* eStop = reinterpret_cast(stopEvent); - command->enqueue(); - - if (startEvent != nullptr) { - eStart->addMarker(queue, command, false); - command->retain(); - } - if (stopEvent != nullptr) { - eStop->addMarker(queue, command, true); - command->retain(); - } - command->release(); - - return hipSuccess; -} - -hipError_t hipModuleLaunchKernel(hipFunction_t f, - uint32_t gridDimX, uint32_t gridDimY, uint32_t gridDimZ, - uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ, - uint32_t sharedMemBytes, hipStream_t hStream, - void **kernelParams, void **extra) -{ - HIP_INIT_API(hipModuleLaunchKernel, f, gridDimX, gridDimY, gridDimZ, - blockDimX, blockDimY, blockDimZ, - sharedMemBytes, hStream, - kernelParams, extra); - size_t globalWorkSizeX = static_cast(gridDimX) * blockDimX; - size_t globalWorkSizeY = static_cast(gridDimY) * blockDimY; - size_t globalWorkSizeZ = static_cast(gridDimZ) * blockDimZ; - if (globalWorkSizeX > std::numeric_limits::max() || - globalWorkSizeY > std::numeric_limits::max() || - globalWorkSizeZ > std::numeric_limits::max()) { - HIP_RETURN(hipErrorInvalidConfiguration); - } - HIP_RETURN(ihipModuleLaunchKernel(f, static_cast(globalWorkSizeX), - static_cast(globalWorkSizeY), - static_cast(globalWorkSizeZ), - blockDimX, blockDimY, blockDimZ, - sharedMemBytes, hStream, kernelParams, extra, nullptr, nullptr)); -} - -hipError_t hipExtModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX, - uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, - uint32_t localWorkSizeX, uint32_t localWorkSizeY, - uint32_t localWorkSizeZ, size_t sharedMemBytes, - hipStream_t hStream, void** kernelParams, void** extra, - hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags) -{ - HIP_INIT_API(hipExtModuleLaunchKernel, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, - localWorkSizeX, localWorkSizeY, localWorkSizeZ, - sharedMemBytes, hStream, - kernelParams, extra, startEvent, stopEvent, flags); - - HIP_RETURN(ihipModuleLaunchKernel(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, localWorkSizeX, localWorkSizeY, - localWorkSizeZ, sharedMemBytes, hStream, kernelParams, extra, startEvent, stopEvent, flags)); -} - - - -hipError_t hipHccModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX, - uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, - uint32_t blockDimX, uint32_t blockDimY, - uint32_t blockDimZ, size_t sharedMemBytes, - hipStream_t hStream, void** kernelParams, void** extra, - hipEvent_t startEvent, - hipEvent_t stopEvent) -{ - HIP_INIT_API(hipHccModuleLaunchKernel, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, - blockDimX, blockDimY, blockDimZ, - sharedMemBytes, hStream, - kernelParams, extra, startEvent, stopEvent); - - HIP_RETURN(ihipModuleLaunchKernel(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, blockDimX, blockDimY, blockDimZ, - sharedMemBytes, hStream, kernelParams, extra, startEvent, stopEvent)); -} - -hipError_t hipModuleLaunchKernelExt(hipFunction_t f, uint32_t globalWorkSizeX, - uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, - uint32_t blockDimX, uint32_t blockDimY, - uint32_t blockDimZ, size_t sharedMemBytes, - hipStream_t hStream, void** kernelParams, void** extra, - hipEvent_t startEvent, - hipEvent_t stopEvent) -{ - HIP_INIT_API(hipModuleLaunchKernelExt, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, - blockDimX, blockDimY, blockDimZ, - sharedMemBytes, hStream, - kernelParams, extra, startEvent, stopEvent); - - HIP_RETURN(ihipModuleLaunchKernel(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, blockDimX, blockDimY, blockDimZ, - sharedMemBytes, hStream, kernelParams, extra, startEvent, stopEvent)); -} - -extern "C" hipError_t hipLaunchKernel(const void *hostFunction, - dim3 gridDim, - dim3 blockDim, - void** args, - size_t sharedMemBytes, - hipStream_t stream) -{ - HIP_INIT_API(hipLaunchKernel, hostFunction, gridDim, blockDim, args, sharedMemBytes, stream); - STREAM_CAPTURE(hipLaunchKernel, stream, hostFunction, gridDim, blockDim, args, sharedMemBytes); - HIP_RETURN(ihipLaunchKernel(hostFunction, gridDim, blockDim, args, sharedMemBytes, stream, - nullptr, nullptr, 0)); -} - -extern "C" hipError_t hipExtLaunchKernel(const void* hostFunction, - dim3 gridDim, - dim3 blockDim, - void** args, - size_t sharedMemBytes, - hipStream_t stream, - hipEvent_t startEvent, - hipEvent_t stopEvent, - int flags) -{ - HIP_INIT_API(hipExtLaunchKernel, hostFunction, gridDim, blockDim, args, sharedMemBytes, stream); - HIP_RETURN(ihipLaunchKernel(hostFunction, gridDim, blockDim, args, sharedMemBytes, stream, startEvent, stopEvent, flags)); -} - -hipError_t hipLaunchCooperativeKernel(const void* f, - dim3 gridDim, dim3 blockDim, - void **kernelParams, uint32_t sharedMemBytes, hipStream_t hStream) -{ - HIP_INIT_API(hipLaunchCooperativeKernel, f, gridDim, blockDim, - sharedMemBytes, hStream); - - hipFunction_t func = nullptr; - int deviceId = hip::Stream::DeviceId(hStream); - HIP_RETURN_ONFAIL(PlatformState::instance().getStatFunc(&func, f, deviceId)); - size_t globalWorkSizeX = static_cast(gridDim.x) * blockDim.x; - size_t globalWorkSizeY = static_cast(gridDim.y) * blockDim.y; - size_t globalWorkSizeZ = static_cast(gridDim.z) * blockDim.z; - if (globalWorkSizeX > std::numeric_limits::max() || - globalWorkSizeY > std::numeric_limits::max() || - globalWorkSizeZ > std::numeric_limits::max()) { - HIP_RETURN(hipErrorInvalidConfiguration); - } - HIP_RETURN(ihipModuleLaunchKernel(func, static_cast(globalWorkSizeX), - static_cast(globalWorkSizeY), - static_cast(globalWorkSizeZ), - blockDim.x, blockDim.y, blockDim.z, - sharedMemBytes, hStream, kernelParams, nullptr, nullptr, nullptr, 0, - amd::NDRangeKernelCommand::CooperativeGroups)); -} - -hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList, - int numDevices, unsigned int flags, uint32_t extFlags) -{ - int numActiveGPUs = 0; - ihipDeviceGetCount(&numActiveGPUs); - - if ((numDevices > numActiveGPUs) || (launchParamsList == nullptr)) { - return hipErrorInvalidValue; - } - - hipError_t result = hipErrorUnknown; - uint64_t allGridSize = 0; - uint32_t blockDims = 0; - std::vector mgpu_list(numDevices); - - for (int i = 0; i < numDevices; ++i) { - const hipLaunchParams& launch = launchParamsList[i]; - blockDims = launch.blockDim.x * launch.blockDim.y * launch.blockDim.z; - allGridSize += launch.gridDim.x * launch.gridDim.y * launch.gridDim.z * - blockDims; - - // Make sure block dimensions are valid - if (0 == blockDims) { - return hipErrorInvalidConfiguration; - } - if (launch.stream != nullptr) { - // Validate devices to make sure it dosn't have duplicates - amd::HostQueue* queue = reinterpret_cast(launch.stream)->asHostQueue(); - auto device = &queue->vdev()->device(); - for (int j = 0; j < numDevices; ++j) { - if (mgpu_list[j] == device) { - return hipErrorInvalidDevice; - } - } - mgpu_list[i] = device; - } else { - return hipErrorInvalidResourceHandle; - } - } - uint64_t prevGridSize = 0; - uint32_t firstDevice = 0; - - // Sync the execution streams on all devices - if ((flags & hipCooperativeLaunchMultiDeviceNoPreSync) == 0) { - for (int i = 0; i < numDevices; ++i) { - amd::HostQueue* queue = - reinterpret_cast(launchParamsList[i].stream)->asHostQueue(); - queue->finish(); - } - } - - for (int i = 0; i < numDevices; ++i) { - const hipLaunchParams& launch = launchParamsList[i]; - amd::HostQueue* queue = reinterpret_cast(launch.stream)->asHostQueue(); - hipFunction_t func = nullptr; - // The order of devices in the launch may not match the order in the global array - for (size_t dev = 0; dev < g_devices.size(); ++dev) { - // Find the matching device and request the kernel function - if (&queue->vdev()->device() == g_devices[dev]->devices()[0]) { - IHIP_RETURN_ONFAIL(PlatformState::instance().getStatFunc(&func, launch.func, dev)); - // Save ROCclr index of the first device in the launch - if (i == 0) { - firstDevice = queue->vdev()->device().index(); - } - break; - } - } - if (func == nullptr) { - result = hipErrorInvalidDeviceFunction; - HIP_RETURN(result); - } - size_t globalWorkSizeX = static_cast(launch.gridDim.x) * launch.blockDim.x; - size_t globalWorkSizeY = static_cast(launch.gridDim.y) * launch.blockDim.y; - size_t globalWorkSizeZ = static_cast(launch.gridDim.z) * launch.blockDim.z; - if (globalWorkSizeX > std::numeric_limits::max() || - globalWorkSizeY > std::numeric_limits::max() || - globalWorkSizeZ > std::numeric_limits::max()) { - HIP_RETURN(hipErrorInvalidConfiguration); - } - result = ihipModuleLaunchKernel(func, static_cast(globalWorkSizeX), - static_cast(globalWorkSizeY), static_cast(globalWorkSizeZ), - launch.blockDim.x, launch.blockDim.y, launch.blockDim.z, - launch.sharedMem, launch.stream, launch.args, nullptr, nullptr, nullptr, - flags, extFlags, i, numDevices, prevGridSize, allGridSize, firstDevice); - if (result != hipSuccess) { - break; - } - prevGridSize += globalWorkSizeX * globalWorkSizeY * globalWorkSizeZ; - } - - // Sync the execution streams on all devices - if ((flags & hipCooperativeLaunchMultiDeviceNoPostSync) == 0) { - for (int i = 0; i < numDevices; ++i) { - amd::HostQueue* queue = - reinterpret_cast(launchParamsList[i].stream)->asHostQueue(); - queue->finish(); - } - } - - return result; -} - -hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList, - int numDevices, unsigned int flags) -{ - HIP_INIT_API(hipLaunchCooperativeKernelMultiDevice, launchParamsList, numDevices, flags); - - return HIP_RETURN(ihipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags, - (amd::NDRangeKernelCommand::CooperativeGroups | - amd::NDRangeKernelCommand::CooperativeMultiDeviceGroups))); -} - -hipError_t hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, - int numDevices, unsigned int flags) { - HIP_INIT_API(hipExtLaunchMultiKernelMultiDevice, launchParamsList, numDevices, flags); - - return HIP_RETURN(ihipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags, 0)); -} - -hipError_t hipModuleGetTexRef(textureReference** texRef, hipModule_t hmod, const char* name) { - HIP_INIT_API(hipModuleGetTexRef, texRef, hmod, name); - - /* input args check */ - if ((texRef == nullptr) || (name == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - /* Get address and size for the global symbol */ - if (hipSuccess != PlatformState::instance().getDynTexRef(name, hmod, texRef)) { - LogPrintfError("Cannot get texRef for name: %s at module:0x%x \n", name, hmod); - HIP_RETURN(hipErrorNotFound); - } - - // Texture references created by HIP driver API - // have the default read mode set to normalized float. - (*texRef)->readMode = hipReadModeNormalizedFloat; - - PlatformState::instance().registerTexRef(*texRef, hmod, std::string(name)); - - HIP_RETURN(hipSuccess); -} diff --git a/src/hipamd/src/hip_peer.cpp b/src/hipamd/src/hip_peer.cpp deleted file mode 100755 index 74c30fb411..0000000000 --- a/src/hipamd/src/hip_peer.cpp +++ /dev/null @@ -1,250 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include - -#include "hip_internal.hpp" - -hipError_t hipDeviceCanAccessPeer(int* canAccessPeer, hipCtx_t thisCtx, hipCtx_t peerCtx) { - HIP_INIT_API(NONE, canAccessPeer, thisCtx, peerCtx); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipMemcpyPeer(void* dst, hipCtx_t dstCtx, const void* src, hipCtx_t srcCtx, - size_t sizeBytes) { - HIP_INIT_API(NONE, dst, dstCtx, src, srcCtx, sizeBytes); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipMemcpyPeerAsync(void* dst, hipCtx_t dstDevice, const void* src, hipCtx_t srcDevice, - size_t sizeBytes, hipStream_t stream) { - HIP_INIT_API(NONE, dst, dstDevice, src, srcDevice, sizeBytes, stream); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t canAccessPeer(int* canAccessPeer, int deviceId, int peerDeviceId){ - amd::Device* device = nullptr; - amd::Device* peer_device = nullptr; - if (canAccessPeer == nullptr) { - return hipErrorInvalidValue; - } - /* Peer cannot be self */ - if (deviceId == peerDeviceId) { - *canAccessPeer = 0; - return hipSuccess; - } - /* Cannot exceed the max number of devices */ - if (static_cast(deviceId) >= g_devices.size() - || static_cast(peerDeviceId) >= g_devices.size()) { - return hipErrorInvalidDevice; - } - device = g_devices[deviceId]->devices()[0]; - peer_device = g_devices[peerDeviceId]->devices()[0]; - *canAccessPeer = static_cast(std::find(device->p2pDevices_.begin(), - device->p2pDevices_.end(), as_cl(peer_device)) - != device->p2pDevices_.end()); - return hipSuccess; -} - -hipError_t findLinkInfo(int device1, int device2, - std::vector* link_attrs) { - - amd::Device* amd_dev_obj1 = nullptr; - amd::Device* amd_dev_obj2 = nullptr; - const int numDevices = static_cast(g_devices.size()); - - if ((device1 < 0) || (device1 >= numDevices) || (device2 < 0) || (device2 >= numDevices)) { - return hipErrorInvalidDevice; - } - - amd_dev_obj1 = g_devices[device1]->devices()[0]; - amd_dev_obj2 = g_devices[device2]->devices()[0]; - - if (!amd_dev_obj1->findLinkInfo(*amd_dev_obj2, link_attrs)) { - return hipErrorInvalidHandle; - } - - return hipSuccess; -} - -hipError_t hipExtGetLinkTypeAndHopCount(int device1, int device2, - uint32_t* linktype, uint32_t* hopcount) { - HIP_INIT_API(hipExtGetLinkTypeAndHopCount, device1, device2, linktype, hopcount); - - if (linktype == nullptr || hopcount == nullptr || - device1 == device2 || device1 < 0 || device2 < 0) { - HIP_RETURN(hipErrorInvalidValue); - } - // Fill out the list of LinkAttributes - std::vector link_attrs; - link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0)); - link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkHopCount, 0)); - - HIP_RETURN_ONFAIL(findLinkInfo(device1, device2, &link_attrs)); - - *linktype = static_cast(link_attrs[0].second); - *hopcount = static_cast(link_attrs[1].second); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceGetP2PAttribute(int* value, hipDeviceP2PAttr attr, - int srcDevice, int dstDevice) { - HIP_INIT_API(hipDeviceGetP2PAttribute, value, attr, srcDevice, dstDevice); - - if (value == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - if (srcDevice == dstDevice || srcDevice >= static_cast(g_devices.size()) - || dstDevice >= static_cast(g_devices.size())) { - HIP_RETURN(hipErrorInvalidDevice); - } - - std::vector link_attrs; - - switch (attr) { - case hipDevP2PAttrPerformanceRank : { - link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0)); - break; - } - case hipDevP2PAttrAccessSupported : { - HIP_RETURN_ONFAIL(canAccessPeer(value, srcDevice, dstDevice)); - break; - } - case hipDevP2PAttrNativeAtomicSupported : { - link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0)); - break; - } - case hipDevP2PAttrHipArrayAccessSupported : { - hipDeviceProp_t srcDeviceProp; - hipDeviceProp_t dstDeviceProp; - HIP_RETURN_ONFAIL(hipGetDeviceProperties(&srcDeviceProp, srcDevice)); - HIP_RETURN_ONFAIL(hipGetDeviceProperties(&dstDeviceProp, dstDevice)); - - // Linear layout access is supported if P2P is enabled - // Opaque Images are supported only on homogeneous systems - // Might have more conditions to check, in future. - if (srcDeviceProp.gcnArch == dstDeviceProp.gcnArch) { - HIP_RETURN_ONFAIL(canAccessPeer(value, srcDevice, dstDevice)); - } else { - value = 0; - } - break; - } - default : { - LogPrintfError("Invalid attribute attr: %d ", attr); - HIP_RETURN(hipErrorInvalidValue); - break; - } - } - - if ((attr != hipDevP2PAttrAccessSupported) && (attr != hipDevP2PAttrHipArrayAccessSupported)) { - HIP_RETURN_ONFAIL(findLinkInfo(srcDevice, dstDevice, &link_attrs)); - *value = static_cast(link_attrs[0].second); - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipDeviceCanAccessPeer(int* canAccess, int deviceId, int peerDeviceId) { - HIP_INIT_API(hipDeviceCanAccessPeer, canAccess, deviceId, peerDeviceId); - HIP_RETURN(canAccessPeer(canAccess, deviceId, peerDeviceId)); -} - -hipError_t hipDeviceDisablePeerAccess(int peerDeviceId) { - HIP_INIT_API(hipDeviceDisablePeerAccess, peerDeviceId); - int deviceId = hip::getCurrentDevice()->deviceId(); - int canAccess = 0; - if ((hipSuccess != canAccessPeer(&canAccess, deviceId, peerDeviceId)) || (canAccess == 0)) { - HIP_RETURN(hipErrorInvalidDevice); - } - - amd::Device* device = g_devices[deviceId]->devices()[0]; - amd::Device* peer_device = g_devices[peerDeviceId]->devices()[0]; - peer_device->disableP2P(device); - - HIP_RETURN(hip::getCurrentDevice()->DisablePeerAccess(peerDeviceId)); -} - -hipError_t hipDeviceEnablePeerAccess(int peerDeviceId, unsigned int flags) { - HIP_INIT_API(hipDeviceEnablePeerAccess, peerDeviceId, flags); - int deviceId = hip::getCurrentDevice()->deviceId(); - int canAccess = 0; - if (flags != 0) { - HIP_RETURN(hipErrorInvalidValue); - } - if ((hipSuccess != canAccessPeer(&canAccess, deviceId, peerDeviceId)) || (canAccess == 0)) { - HIP_RETURN(hipErrorInvalidDevice); - } - - amd::Device* device = g_devices[deviceId]->asContext()->devices()[0]; - amd::Device* peer_device = g_devices[peerDeviceId]->asContext()->devices()[0]; - peer_device->enableP2P(device); - - HIP_RETURN(hip::getCurrentDevice()->EnablePeerAccess(peerDeviceId)); -} - -hipError_t hipMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, - size_t sizeBytes) { - HIP_INIT_API(hipMemcpyPeer, dst, dstDevice, src, srcDevice, sizeBytes); - - if (srcDevice >= static_cast(g_devices.size()) || - dstDevice >= static_cast(g_devices.size()) || - srcDevice < 0 || dstDevice < 0) { - HIP_RETURN(hipErrorInvalidDevice); - } - - HIP_RETURN(hipMemcpy(dst, src, sizeBytes, hipMemcpyDeviceToDevice)); -} - -hipError_t hipMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, - size_t sizeBytes, hipStream_t stream) { - HIP_INIT_API(hipMemcpyPeerAsync, dst, dstDevice, src, srcDevice, sizeBytes, stream); - - if (srcDevice >= static_cast(g_devices.size()) || - dstDevice >= static_cast(g_devices.size()) || - srcDevice < 0 || dstDevice < 0) { - HIP_RETURN(hipErrorInvalidDevice); - } - - HIP_RETURN(hipMemcpyAsync(dst, src, sizeBytes, hipMemcpyDeviceToDevice, stream)); -} - -hipError_t hipCtxEnablePeerAccess(hipCtx_t peerCtx, unsigned int flags) { - HIP_INIT_API(hipCtxEnablePeerAccess, peerCtx, flags); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipCtxDisablePeerAccess(hipCtx_t peerCtx) { - HIP_INIT_API(hipCtxDisablePeerAccess, peerCtx); - - HIP_RETURN(hipSuccess); -} diff --git a/src/hipamd/src/hip_platform.cpp b/src/hipamd/src/hip_platform.cpp deleted file mode 100755 index 80ffb2c657..0000000000 --- a/src/hipamd/src/hip_platform.cpp +++ /dev/null @@ -1,945 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include -#include "hip_platform.hpp" -#include "hip_internal.hpp" -#include "platform/program.hpp" -#include "platform/runtime.hpp" - -#include - -constexpr unsigned __hipFatMAGIC2 = 0x48495046; // "HIPF" - -thread_local std::stack execStack_; -PlatformState* PlatformState::platform_; // Initiaized as nullptr by default - -//forward declaration of methods required for __hipRegisrterManagedVar -hipError_t ihipMallocManaged(void** ptr, size_t size, unsigned int align = 0); -hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind, - amd::HostQueue& queue, bool isAsync = false); - -struct __CudaFatBinaryWrapper { - unsigned int magic; - unsigned int version; - void* binary; - void* dummy1; -}; - -hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes, - hipModule_t hmod, const char* name); - -hipError_t ihipCreateGlobalVarObj(const char* name, hipModule_t hmod, amd::Memory** amd_mem_obj, - hipDeviceptr_t* dptr, size_t* bytes); - -extern hipError_t ihipModuleLaunchKernel(hipFunction_t f, - uint32_t gridDimX, uint32_t gridDimY, uint32_t gridDimZ, - uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ, - uint32_t sharedMemBytes, hipStream_t hStream, - void **kernelParams, void **extra, - hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags = 0, - uint32_t params = 0, uint32_t gridId = 0, uint32_t numGrids = 0, - uint64_t prevGridSum = 0, uint64_t allGridSum = 0, uint32_t firstDevice = 0); -static bool isCompatibleCodeObject(const std::string& codeobj_target_id, - const char* device_name) { - // Workaround for device name mismatch. - // Device name may contain feature strings delimited by '+', e.g. - // gfx900+xnack. Currently HIP-Clang does not include feature strings - // in code object target id in fat binary. Therefore drop the feature - // strings from device name before comparing it with code object target id. - std::string short_name(device_name); - auto feature_loc = short_name.find('+'); - if (feature_loc != std::string::npos) { - short_name.erase(feature_loc); - } - return codeobj_target_id == short_name; -} - -extern "C" hip::FatBinaryInfo** __hipRegisterFatBinary(const void* data) -{ - const __CudaFatBinaryWrapper* fbwrapper = reinterpret_cast(data); - if (fbwrapper->magic != __hipFatMAGIC2 || fbwrapper->version != 1) { - LogPrintfError("Cannot Register fat binary. FatMagic: %u version: %u ", fbwrapper->magic, - fbwrapper->version); - return nullptr; - } - return PlatformState::instance().addFatBinary(fbwrapper->binary); -} - -extern "C" void __hipRegisterFunction( - hip::FatBinaryInfo** modules, - const void* hostFunction, - char* deviceFunction, - const char* deviceName, - unsigned int threadLimit, - uint3* tid, - uint3* bid, - dim3* blockDim, - dim3* gridDim, - int* wSize) { - static int enable_deferred_loading { []() { - char *var = getenv("HIP_ENABLE_DEFERRED_LOADING"); - return var ? atoi(var) : 1; - }() }; - - hip::Function* func = new hip::Function(std::string(deviceName), modules); - PlatformState::instance().registerStatFunction(hostFunction, func); - - if (!enable_deferred_loading) { - HIP_INIT(); - hipFunction_t hfunc = nullptr; - hipError_t hip_error = hipSuccess; - for (size_t dev_idx = 0; dev_idx < g_devices.size(); ++dev_idx) { - hip_error = PlatformState::instance().getStatFunc(&hfunc, hostFunction, dev_idx); - guarantee((hip_error == hipSuccess), "Cannot Retrieve Static function"); - } - } -} - -// Registers a device-side global variable. -// For each global variable in device code, there is a corresponding shadow -// global variable in host code. The shadow host variable is used to keep -// track of the value of the device side global variable between kernel -// executions. -extern "C" void __hipRegisterVar( - hip::FatBinaryInfo** modules, // The device modules containing code object - void* var, // The shadow variable in host code - char* hostVar, // Variable name in host code - char* deviceVar, // Variable name in device code - int ext, // Whether this variable is external - size_t size, // Size of the variable - int constant, // Whether this variable is constant - int global) // Unknown, always 0 -{ - hip::Var* var_ptr = new hip::Var(std::string(hostVar), hip::Var::DeviceVarKind::DVK_Variable, size, 0, 0, modules); - PlatformState::instance().registerStatGlobalVar(var, var_ptr); -} - -extern "C" void __hipRegisterSurface(hip::FatBinaryInfo** modules, // The device modules containing code object - void* var, // The shadow variable in host code - char* hostVar, // Variable name in host code - char* deviceVar, // Variable name in device code - int type, int ext) { - hip::Var* var_ptr = new hip::Var(std::string(hostVar), hip::Var::DeviceVarKind::DVK_Surface, sizeof(surfaceReference), 0, 0, modules); - PlatformState::instance().registerStatGlobalVar(var, var_ptr); -} - -extern "C" void __hipRegisterManagedVar(void *hipModule, // Pointer to hip module returned from __hipRegisterFatbinary - void **pointer, // Pointer to a chunk of managed memory with size \p size and alignment \p align - // HIP runtime allocates such managed memory and assign it to \p pointer - void *init_value, // Initial value to be copied into \p pointer - const char *name, // Name of the variable in code object - size_t size, - unsigned align) { - HIP_INIT(); - hipError_t status = ihipMallocManaged(pointer, size, align); - if( status == hipSuccess) { - amd::HostQueue* queue = hip::getNullStream(); - if(queue != nullptr) { - ihipMemcpy(*pointer, init_value, size, hipMemcpyHostToDevice, *queue); - } else { - ClPrint(amd::LOG_ERROR, amd::LOG_API, "Host Queue is NULL"); - } - } else { - guarantee(false, "Error during allocation of managed memory!"); - } - hip::Var* var_ptr = new hip::Var(std::string(name), hip::Var::DeviceVarKind::DVK_Managed, pointer, - size, align, reinterpret_cast(hipModule)); - PlatformState::instance().registerStatManagedVar(var_ptr); -} - -extern "C" void __hipRegisterTexture(hip::FatBinaryInfo** modules, // The device modules containing code object - void* var, // The shadow variable in host code - char* hostVar, // Variable name in host code - char* deviceVar, // Variable name in device code - int type, int norm, int ext) { - hip::Var* var_ptr = new hip::Var(std::string(hostVar), hip::Var::DeviceVarKind::DVK_Texture, sizeof(textureReference), 0, 0, modules); - PlatformState::instance().registerStatGlobalVar(var, var_ptr); -} - -extern "C" void __hipUnregisterFatBinary(hip::FatBinaryInfo** modules) -{ - PlatformState::instance().removeFatBinary(modules); -} - -extern "C" hipError_t hipConfigureCall( - dim3 gridDim, - dim3 blockDim, - size_t sharedMem, - hipStream_t stream) -{ - HIP_INIT_API(hipConfigureCall, gridDim, blockDim, sharedMem, stream); - - PlatformState::instance().configureCall(gridDim, blockDim, sharedMem, stream); - - HIP_RETURN(hipSuccess); -} - -extern "C" hipError_t __hipPushCallConfiguration( - dim3 gridDim, - dim3 blockDim, - size_t sharedMem, - hipStream_t stream) -{ - HIP_INIT_API(__hipPushCallConfiguration, gridDim, blockDim, sharedMem, stream); - - PlatformState::instance().configureCall(gridDim, blockDim, sharedMem, stream); - - HIP_RETURN(hipSuccess); -} - -extern "C" hipError_t __hipPopCallConfiguration(dim3 *gridDim, - dim3 *blockDim, - size_t *sharedMem, - hipStream_t *stream) { - HIP_INIT_API(__hipPopCallConfiguration, gridDim, blockDim, sharedMem, stream); - - ihipExec_t exec; - PlatformState::instance().popExec(exec); - *gridDim = exec.gridDim_; - *blockDim = exec.blockDim_; - *sharedMem = exec.sharedMem_; - *stream = exec.hStream_; - - HIP_RETURN(hipSuccess); -} - -extern "C" hipError_t hipSetupArgument( - const void *arg, - size_t size, - size_t offset) -{ - HIP_INIT_API(hipSetupArgument, arg, size, offset); - - PlatformState::instance().setupArgument(arg, size, offset); - - HIP_RETURN(hipSuccess); -} - -extern "C" hipError_t hipLaunchByPtr(const void *hostFunction) -{ - HIP_INIT_API(hipLaunchByPtr, hostFunction); - - ihipExec_t exec; - PlatformState::instance().popExec(exec); - - hip::Stream* stream = reinterpret_cast(exec.hStream_); - int deviceId = (stream != nullptr)? stream->DeviceId() : ihipGetDevice(); - if (deviceId == -1) { - LogPrintfError("Wrong DeviceId: %d \n", deviceId); - HIP_RETURN(hipErrorNoDevice); - } - hipFunction_t func = nullptr; - hipError_t hip_error = PlatformState::instance().getStatFunc(&func, hostFunction, deviceId); - if ((hip_error != hipSuccess) || (func == nullptr)) { - LogPrintfError("Could not retrieve hostFunction: 0x%x \n", hostFunction); - HIP_RETURN(hipErrorInvalidDeviceFunction); - } - - size_t size = exec.arguments_.size(); - void *extra[] = { - HIP_LAUNCH_PARAM_BUFFER_POINTER, &exec.arguments_[0], - HIP_LAUNCH_PARAM_BUFFER_SIZE, &size, - HIP_LAUNCH_PARAM_END - }; - - HIP_RETURN(hipModuleLaunchKernel(func, - exec.gridDim_.x, exec.gridDim_.y, exec.gridDim_.z, - exec.blockDim_.x, exec.blockDim_.y, exec.blockDim_.z, - exec.sharedMem_, exec.hStream_, nullptr, extra)); -} - -hipError_t hipGetSymbolAddress(void** devPtr, const void* symbol) { - HIP_INIT_API(hipGetSymbolAddress, devPtr, symbol); - - hipError_t hip_error = hipSuccess; - size_t sym_size = 0; - - HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(symbol, ihipGetDevice(), devPtr, &sym_size)); - - HIP_RETURN(hipSuccess, *devPtr); -} - -hipError_t hipGetSymbolSize(size_t* sizePtr, const void* symbol) { - HIP_INIT_API(hipGetSymbolSize, sizePtr, symbol); - - hipDeviceptr_t device_ptr = nullptr; - HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(symbol, ihipGetDevice(), &device_ptr, sizePtr)); - - HIP_RETURN(hipSuccess, *sizePtr); -} - -hipError_t ihipCreateGlobalVarObj(const char* name, hipModule_t hmod, amd::Memory** amd_mem_obj, - hipDeviceptr_t* dptr, size_t* bytes) -{ - HIP_INIT(); - - amd::Program* program = nullptr; - device::Program* dev_program = nullptr; - - /* Get Device Program pointer*/ - program = as_amd(reinterpret_cast(hmod)); - dev_program = program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]); - - if (dev_program == nullptr) { - LogPrintfError("Cannot get Device Function for module: 0x%x \n", hmod); - HIP_RETURN(hipErrorInvalidDeviceFunction); - } - /* Find the global Symbols */ - if (!dev_program->createGlobalVarObj(amd_mem_obj, dptr, bytes, name)) { - LogPrintfError("Cannot create Global Var obj for symbol: %s \n", name); - HIP_RETURN(hipErrorInvalidSymbol); - } - - HIP_RETURN(hipSuccess); -} - - -namespace hip_impl { -hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor( - int* maxBlocksPerCU, int* numBlocksPerGrid, int* bestBlockSize, - const amd::Device& device, hipFunction_t func, int inputBlockSize, - size_t dynamicSMemSize, bool bCalcPotentialBlkSz) -{ - hip::DeviceFunc* function = hip::DeviceFunc::asFunction(func); - const amd::Kernel& kernel = *function->kernel(); - - const device::Kernel::WorkGroupInfo* wrkGrpInfo = kernel.getDeviceKernel(device)->workGroupInfo(); - if (bCalcPotentialBlkSz == false) { - if (inputBlockSize <= 0) { - return hipErrorInvalidValue; - } - *bestBlockSize = 0; - // Make sure the requested block size is smaller than max supported - if (inputBlockSize > int(device.info().maxWorkGroupSize_)) { - *maxBlocksPerCU = 0; - *numBlocksPerGrid = 0; - return hipSuccess; - } - } - else { - if (inputBlockSize > int(device.info().maxWorkGroupSize_) || - inputBlockSize <= 0) { - // The user wrote the kernel to work with a workgroup size - // bigger than this hardware can support. Or they do not care - // about the size So just assume its maximum size is - // constrained by hardware - inputBlockSize = device.info().maxWorkGroupSize_; - } - } - // Find wave occupancy per CU => simd_per_cu * GPR usage - size_t MaxWavesPerSimd; - - if (device.isa().versionMajor() <= 9) { - MaxWavesPerSimd = 8; // Limited by SPI 32 per CU, hence 8 per SIMD - } else { - MaxWavesPerSimd = 16; - } - size_t VgprWaves = MaxWavesPerSimd; - size_t maxVGPRs; - uint32_t VgprGranularity; - if (device.isa().versionMajor() <= 9) { - if (device.isa().versionMajor() == 9 && - device.isa().versionMinor() == 0 && - device.isa().versionStepping() == 10) { - maxVGPRs = 512; - VgprGranularity = 8; - } - else { - maxVGPRs = 256; - VgprGranularity = 4; - } - } - else { - maxVGPRs = 1024; - VgprGranularity = 8; - } - if (wrkGrpInfo->usedSGPRs_ > 0) { - VgprWaves = maxVGPRs / amd::alignUp(wrkGrpInfo->usedVGPRs_, VgprGranularity); - } - - size_t GprWaves = VgprWaves; - if (wrkGrpInfo->usedSGPRs_ > 0) { - size_t maxSGPRs; - if (device.isa().versionMajor() < 8) { - maxSGPRs = 512; - } - else if (device.isa().versionMajor() < 10) { - maxSGPRs = 800; - } - else { - maxSGPRs = SIZE_MAX; // gfx10+ does not share SGPRs between waves - } - const size_t SgprWaves = maxSGPRs / amd::alignUp(wrkGrpInfo->usedSGPRs_, 16); - GprWaves = std::min(VgprWaves, SgprWaves); - } - - const size_t alu_occupancy = device.info().simdPerCU_ * std::min(MaxWavesPerSimd, GprWaves); - const int alu_limited_threads = alu_occupancy * wrkGrpInfo->wavefrontSize_; - - int lds_occupancy_wgs = INT_MAX; - const size_t total_used_lds = wrkGrpInfo->usedLDSSize_ + dynamicSMemSize; - if (total_used_lds != 0) { - lds_occupancy_wgs = static_cast(device.info().localMemSize_ / total_used_lds); - } - // Calculate how many blocks of inputBlockSize we can fit per CU - // Need to align with hardware wavefront size. If they want 65 threads, but - // waves are 64, then we need 128 threads per block. - // So this calculates how many blocks we can fit. - *maxBlocksPerCU = alu_limited_threads / amd::alignUp(inputBlockSize, wrkGrpInfo->wavefrontSize_); - // Unless those blocks are further constrained by LDS size. - *maxBlocksPerCU = std::min(*maxBlocksPerCU, lds_occupancy_wgs); - - // Some callers of this function want to return the block size, in threads, that - // leads to the maximum occupancy. In that case, inputBlockSize is the maximum - // workgroup size the user wants to allow, or that the hardware can allow. - // It is either the number of threads that we are limited to due to occupancy, or - // the maximum available block size for this kernel, which could have come from the - // user. e.g., if the user indicates the maximum block size is 64 threads, but we - // calculate that 128 threads can fit in each CU, we have to give up and return 64. - *bestBlockSize = std::min(alu_limited_threads, amd::alignUp(inputBlockSize, wrkGrpInfo->wavefrontSize_)); - // If the best block size is smaller than the block size used to fit the maximum, - // then we need to make the grid bigger for full occupancy. - const int bestBlocksPerCU = alu_limited_threads / (*bestBlockSize); - // Unless those blocks are further constrained by LDS size. - *numBlocksPerGrid = device.info().maxComputeUnits_ * std::min(bestBlocksPerCU, lds_occupancy_wgs); - - return hipSuccess; -} -} - -extern "C" { -hipError_t hipOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize, - const void* f, size_t dynSharedMemPerBlk, - int blockSizeLimit) -{ - HIP_INIT_API(hipOccupancyMaxPotentialBlockSize, f, dynSharedMemPerBlk, blockSizeLimit); - if ((gridSize == nullptr) || (blockSize == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - hipFunction_t func = nullptr; - hipError_t hip_error = PlatformState::instance().getStatFunc(&func, f, ihipGetDevice()); - if ((hip_error != hipSuccess) || (func == nullptr)) { - return HIP_RETURN(hipErrorInvalidValue); - } - const amd::Device& device = *hip::getCurrentDevice()->devices()[0]; - int max_blocks_per_grid = 0; - int num_blocks = 0; - int best_block_size = 0; - hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor( - &num_blocks, &max_blocks_per_grid, &best_block_size, device, func, blockSizeLimit, dynSharedMemPerBlk,true); - if (ret == hipSuccess) { - *blockSize = best_block_size; - *gridSize = max_blocks_per_grid; - } - HIP_RETURN(ret); -} - -hipError_t hipModuleOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize, - hipFunction_t f, size_t dynSharedMemPerBlk, - int blockSizeLimit) -{ - HIP_INIT_API(hipModuleOccupancyMaxPotentialBlockSize, f, dynSharedMemPerBlk, blockSizeLimit); - if ((gridSize == nullptr) || (blockSize == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - const amd::Device& device = *hip::getCurrentDevice()->devices()[0]; - int max_blocks_per_grid = 0; - int num_blocks = 0; - int best_block_size = 0; - hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor( - &num_blocks, &max_blocks_per_grid, &best_block_size, device, f, blockSizeLimit, dynSharedMemPerBlk,true); - if (ret == hipSuccess) { - *blockSize = best_block_size; - *gridSize = max_blocks_per_grid; - } - HIP_RETURN(ret); -} - -hipError_t hipModuleOccupancyMaxPotentialBlockSizeWithFlags(int* gridSize, int* blockSize, - hipFunction_t f, size_t dynSharedMemPerBlk, - int blockSizeLimit, unsigned int flags) -{ - HIP_INIT_API(hipModuleOccupancyMaxPotentialBlockSizeWithFlags, f, dynSharedMemPerBlk, blockSizeLimit, flags); - if ((gridSize == nullptr) || (blockSize == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - const amd::Device& device = *hip::getCurrentDevice()->devices()[0]; - int max_blocks_per_grid = 0; - int num_blocks = 0; - int best_block_size = 0; - hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor( - &num_blocks, &max_blocks_per_grid, &best_block_size, device, f, blockSizeLimit, dynSharedMemPerBlk,true); - if (ret == hipSuccess) { - *blockSize = best_block_size; - *gridSize = max_blocks_per_grid; - } - HIP_RETURN(ret); -} - -hipError_t hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, - hipFunction_t f, int blockSize, size_t dynSharedMemPerBlk) -{ - HIP_INIT_API(hipModuleOccupancyMaxActiveBlocksPerMultiprocessor, f, blockSize, dynSharedMemPerBlk); - if (numBlocks == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - const amd::Device& device = *hip::getCurrentDevice()->devices()[0]; - - int num_blocks = 0; - int max_blocks_per_grid = 0; - int best_block_size = 0; - hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor( - &num_blocks, &max_blocks_per_grid, &best_block_size, device, f, blockSize, dynSharedMemPerBlk, false); - *numBlocks = num_blocks; - HIP_RETURN(ret); -} - -hipError_t hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, - hipFunction_t f, int blockSize, - size_t dynSharedMemPerBlk, unsigned int flags) -{ - HIP_INIT_API(hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, f, blockSize, dynSharedMemPerBlk, flags); - if (numBlocks == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - const amd::Device& device = *hip::getCurrentDevice()->devices()[0]; - - int num_blocks = 0; - int max_blocks_per_grid = 0; - int best_block_size = 0; - hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor( - &num_blocks, &max_blocks_per_grid, &best_block_size, device, f, blockSize, dynSharedMemPerBlk, false); - *numBlocks = num_blocks; - HIP_RETURN(ret); -} - -hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, - const void* f, int blockSize, size_t dynamicSMemSize) -{ - HIP_INIT_API(hipOccupancyMaxActiveBlocksPerMultiprocessor, f, blockSize, dynamicSMemSize); - if (numBlocks == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipFunction_t func = nullptr; - hipError_t hip_error = PlatformState::instance().getStatFunc(&func, f, ihipGetDevice()); - if ((hip_error != hipSuccess) || (func == nullptr)) { - return HIP_RETURN(hipErrorInvalidValue); - } - - const amd::Device& device = *hip::getCurrentDevice()->devices()[0]; - - int num_blocks = 0; - int max_blocks_per_grid = 0; - int best_block_size = 0; - hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor( - &num_blocks, &max_blocks_per_grid, &best_block_size, device, func, blockSize, dynamicSMemSize, false); - *numBlocks = num_blocks; - HIP_RETURN(ret); -} - -hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, - const void* f, - int blockSize, size_t dynamicSMemSize, unsigned int flags) -{ - HIP_INIT_API(hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, f, blockSize, dynamicSMemSize, flags); - if (numBlocks == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipFunction_t func = nullptr; - hipError_t hip_error = PlatformState::instance().getStatFunc(&func, f, ihipGetDevice()); - if ((hip_error != hipSuccess) || (func == nullptr)) { - return HIP_RETURN(hipErrorInvalidValue); - } - - const amd::Device& device = *hip::getCurrentDevice()->devices()[0]; - - int num_blocks = 0; - int max_blocks_per_grid = 0; - int best_block_size = 0; - hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor( - &num_blocks, &max_blocks_per_grid, &best_block_size, device, func, blockSize, dynamicSMemSize, false); - *numBlocks = num_blocks; - HIP_RETURN(ret); -} -} - - -#if defined(ATI_OS_LINUX) - -namespace hip_impl { - -void hipLaunchKernelGGLImpl( - uintptr_t function_address, - const dim3& numBlocks, - const dim3& dimBlocks, - uint32_t sharedMemBytes, - hipStream_t stream, - void** kernarg) -{ - HIP_INIT(); - - hip::Stream* s = reinterpret_cast(stream); - int deviceId = (s != nullptr)? s->DeviceId() : ihipGetDevice(); - if (deviceId == -1) { - LogPrintfError("Wrong Device Id: %d \n", deviceId); - } - - hipFunction_t func = nullptr; - hipError_t hip_error = PlatformState::instance().getStatFunc(&func, reinterpret_cast(function_address), deviceId); - if ((hip_error != hipSuccess) || (func == nullptr)) { - LogPrintfError("Cannot find the static function: 0x%x", function_address); - } - - hipModuleLaunchKernel(func, - numBlocks.x, numBlocks.y, numBlocks.z, - dimBlocks.x, dimBlocks.y, dimBlocks.z, - sharedMemBytes, stream, nullptr, kernarg); -} - -void hipLaunchCooperativeKernelGGLImpl( - uintptr_t function_address, - const dim3& numBlocks, - const dim3& dimBlocks, - uint32_t sharedMemBytes, - hipStream_t stream, - void** kernarg) -{ - HIP_INIT(); - - hipLaunchCooperativeKernel(reinterpret_cast(function_address), - numBlocks, dimBlocks, kernarg, sharedMemBytes, stream); -} - -} - -#endif // defined(ATI_OS_LINUX) - -hipError_t ihipLaunchKernel(const void* hostFunction, - dim3 gridDim, - dim3 blockDim, - void** args, - size_t sharedMemBytes, - hipStream_t stream, - hipEvent_t startEvent, - hipEvent_t stopEvent, - int flags) -{ - hipFunction_t func = nullptr; - int deviceId = hip::Stream::DeviceId(stream); - hipError_t hip_error = PlatformState::instance().getStatFunc(&func, hostFunction, deviceId); - if ((hip_error != hipSuccess) || (func == nullptr)) { - HIP_RETURN(hipErrorInvalidDeviceFunction); - } - size_t globalWorkSizeX = static_cast(gridDim.x) * blockDim.x; - size_t globalWorkSizeY = static_cast(gridDim.y) * blockDim.y; - size_t globalWorkSizeZ = static_cast(gridDim.z) * blockDim.z; - if (globalWorkSizeX > std::numeric_limits::max() || - globalWorkSizeY > std::numeric_limits::max() || - globalWorkSizeZ > std::numeric_limits::max()) { - HIP_RETURN(hipErrorInvalidConfiguration); - } - HIP_RETURN(ihipModuleLaunchKernel(func, static_cast(globalWorkSizeX), - static_cast(globalWorkSizeY), - static_cast(globalWorkSizeZ), - blockDim.x, blockDim.y, blockDim.z, - sharedMemBytes, stream, args, nullptr, startEvent, stopEvent, - flags)); -} - -// conversion routines between float and half precision - -static inline std::uint32_t f32_as_u32(float f) { union { float f; std::uint32_t u; } v; v.f = f; return v.u; } - -static inline float u32_as_f32(std::uint32_t u) { union { float f; std::uint32_t u; } v; v.u = u; return v.f; } - -static inline int clamp_int(int i, int l, int h) { return std::min(std::max(i, l), h); } - - -// half float, the f16 is in the low 16 bits of the input argument - -static inline float __convert_half_to_float(std::uint32_t a) noexcept { - - std::uint32_t u = ((a << 13) + 0x70000000U) & 0x8fffe000U; - - std::uint32_t v = f32_as_u32(u32_as_f32(u) * u32_as_f32(0x77800000U)/*0x1.0p+112f*/) + 0x38000000U; - - u = (a & 0x7fff) != 0 ? v : u; - - return u32_as_f32(u) * u32_as_f32(0x07800000U)/*0x1.0p-112f*/; - -} - -// float half with nearest even rounding -// The lower 16 bits of the result is the bit pattern for the f16 -static inline std::uint32_t __convert_float_to_half(float a) noexcept { - std::uint32_t u = f32_as_u32(a); - int e = static_cast((u >> 23) & 0xff) - 127 + 15; - std::uint32_t m = ((u >> 11) & 0xffe) | ((u & 0xfff) != 0); - std::uint32_t i = 0x7c00 | (m != 0 ? 0x0200 : 0); - std::uint32_t n = ((std::uint32_t)e << 12) | m; - std::uint32_t s = (u >> 16) & 0x8000; - int b = clamp_int(1-e, 0, 13); - std::uint32_t d = (0x1000 | m) >> b; - d |= (d << b) != (0x1000 | m); - std::uint32_t v = e < 1 ? d : n; - v = (v >> 2) + (((v & 0x7) == 3) | ((v & 0x7) > 5)); - v = e > 30 ? 0x7c00 : v; - v = e == 143 ? i : v; - return s | v; -} - -extern "C" -#if !defined(_MSC_VER) -__attribute__((weak)) -#endif -float __gnu_h2f_ieee(unsigned short h){ - return __convert_half_to_float((std::uint32_t) h); -} - -extern "C" -#if !defined(_MSC_VER) -__attribute__((weak)) -#endif -unsigned short __gnu_f2h_ieee(float f){ - return (unsigned short)__convert_float_to_half(f); -} - -void PlatformState::init() -{ - amd::ScopedLock lock(lock_); - if(initialized_ || g_devices.empty()) { - return; - } - initialized_ = true; - for (auto& it : statCO_.modules_) { - digestFatBinary(it.first, it.second); - } - for (auto &it : statCO_.vars_) { - it.second->resize_dVar(g_devices.size()); - } - for (auto &it : statCO_.functions_) { - it.second->resize_dFunc(g_devices.size()); - } -} - -hipError_t PlatformState::loadModule(hipModule_t *module, const char* fname, const void* image) { - amd::ScopedLock lock(lock_); - - if(module == nullptr) { - return hipErrorInvalidValue; - } - - hip::DynCO* dynCo = new hip::DynCO(); - hipError_t hip_error = dynCo->loadCodeObject(fname, image); - if (hip_error != hipSuccess) { - delete dynCo; - return hip_error; - } - - *module = dynCo->module(); - assert(*module != nullptr); - - if (dynCO_map_.find(*module) != dynCO_map_.end()) { - return hipErrorAlreadyMapped; - } - dynCO_map_.insert(std::make_pair(*module, dynCo)); - - return hipSuccess; -} - -hipError_t PlatformState::unloadModule(hipModule_t hmod) { - amd::ScopedLock lock(lock_); - - auto it = dynCO_map_.find(hmod); - if (it == dynCO_map_.end()) { - return hipErrorNotFound; - } - - delete it->second; - dynCO_map_.erase(hmod); - - auto tex_it = texRef_map_.begin(); - while (tex_it != texRef_map_.end()) { - if (tex_it->second.first == hmod) { - tex_it = texRef_map_.erase(tex_it); - } else { - ++tex_it; - } - } - - return hipSuccess; -} - -hipError_t PlatformState::getDynFunc(hipFunction_t* hfunc, hipModule_t hmod, - const char* func_name) { - amd::ScopedLock lock(lock_); - - auto it = dynCO_map_.find(hmod); - if (it == dynCO_map_.end()) { - LogPrintfError("Cannot find the module: 0x%x", hmod); - return hipErrorNotFound; - } - if (0 == strlen(func_name)) { - return hipErrorNotFound; - } - - return it->second->getDynFunc(hfunc, func_name); -} - -hipError_t PlatformState::getDynGlobalVar(const char* hostVar, hipModule_t hmod, - hipDeviceptr_t* dev_ptr, size_t* size_ptr) { - amd::ScopedLock lock(lock_); - - if(hostVar == nullptr || dev_ptr == nullptr || size_ptr == nullptr) { - return hipErrorInvalidValue; - } - - auto it = dynCO_map_.find(hmod); - if (it == dynCO_map_.end()) { - LogPrintfError("Cannot find the module: 0x%x", hmod); - return hipErrorNotFound; - } - *dev_ptr = nullptr; - it->second->getManagedVarPointer(hostVar, dev_ptr, size_ptr); - // if dev_ptr is nullptr, hostvar is not in managed variable list - if (*dev_ptr == nullptr) { - hip::DeviceVar* dvar = nullptr; - IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, hostVar)); - *dev_ptr = dvar->device_ptr(); - *size_ptr = dvar->size(); - } - return hipSuccess; -} - -hipError_t PlatformState::registerTexRef(textureReference* texRef, hipModule_t hmod, - std::string name) { - amd::ScopedLock lock(lock_); - texRef_map_.insert(std::make_pair(texRef, std::make_pair(hmod, name))); - return hipSuccess; -} - -hipError_t PlatformState::getDynTexGlobalVar(textureReference* texRef, hipDeviceptr_t* dev_ptr, - size_t* size_ptr) { - amd::ScopedLock lock(lock_); - - auto tex_it = texRef_map_.find(texRef); - if (tex_it == texRef_map_.end()) { - LogPrintfError("Cannot find the texRef Entry: 0x%x", texRef); - return hipErrorNotFound; - } - - auto it = dynCO_map_.find(tex_it->second.first); - if (it == dynCO_map_.end()) { - LogPrintfError("Cannot find the module: 0x%x", tex_it->second.first); - return hipErrorNotFound; - } - - hip::DeviceVar* dvar = nullptr; - IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, tex_it->second.second)); - *dev_ptr = dvar->device_ptr(); - *size_ptr = dvar->size(); - - return hipSuccess; -} - -hipError_t PlatformState::getDynTexRef(const char* hostVar, hipModule_t hmod, textureReference** texRef) { - amd::ScopedLock lock(lock_); - - auto it = dynCO_map_.find(hmod); - if (it == dynCO_map_.end()) { - LogPrintfError("Cannot find the module: 0x%x", hmod); - return hipErrorNotFound; - } - - hip::DeviceVar* dvar = nullptr; - IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, hostVar)); - - dvar->shadowVptr = new texture(); - *texRef = reinterpret_cast(dvar->shadowVptr); - return hipSuccess; -} - -hipError_t PlatformState::digestFatBinary(const void* data, hip::FatBinaryInfo*& programs) { - return statCO_.digestFatBinary(data, programs); -} - -hip::FatBinaryInfo** PlatformState::addFatBinary(const void* data) { - return statCO_.addFatBinary(data, initialized_); -} - -hipError_t PlatformState::removeFatBinary(hip::FatBinaryInfo** module) { - return statCO_.removeFatBinary(module); -} - -hipError_t PlatformState::registerStatFunction(const void* hostFunction, hip::Function* func) { - return statCO_.registerStatFunction(hostFunction, func); -} - -hipError_t PlatformState::registerStatGlobalVar(const void* hostVar, hip::Var* var) { - return statCO_.registerStatGlobalVar(hostVar, var); -} - -hipError_t PlatformState::registerStatManagedVar(hip::Var* var) { - return statCO_.registerStatManagedVar(var); -} - -hipError_t PlatformState::getStatFunc(hipFunction_t* hfunc, const void* hostFunction, int deviceId) { - return statCO_.getStatFunc(hfunc, hostFunction, deviceId); -} - -hipError_t PlatformState::getStatFuncAttr(hipFuncAttributes* func_attr, const void* hostFunction, int deviceId) { - if(func_attr == nullptr || hostFunction == nullptr) { - return hipErrorInvalidValue; - } - return statCO_.getStatFuncAttr(func_attr, hostFunction, deviceId); -} - -hipError_t PlatformState::getStatGlobalVar(const void* hostVar, int deviceId, hipDeviceptr_t* dev_ptr, - size_t* size_ptr) { - return statCO_.getStatGlobalVar(hostVar, deviceId, dev_ptr, size_ptr); -} - -hipError_t PlatformState::initStatManagedVarDevicePtr(int deviceId) { - return statCO_.initStatManagedVarDevicePtr(deviceId); -} - -void PlatformState::setupArgument(const void *arg, size_t size, size_t offset) { - auto& arguments = execStack_.top().arguments_; - - if (arguments.size() < offset + size) { - arguments.resize(offset + size); - } - - ::memcpy(&arguments[offset], arg, size); -} - -void PlatformState::configureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, - hipStream_t stream) { - execStack_.push(ihipExec_t{gridDim, blockDim, sharedMem, stream}); -} - -void PlatformState::popExec(ihipExec_t& exec) { - exec = std::move(execStack_.top()); - execStack_.pop(); -} diff --git a/src/hipamd/src/hip_platform.hpp b/src/hipamd/src/hip_platform.hpp deleted file mode 100755 index 8ab3c1c09e..0000000000 --- a/src/hipamd/src/hip_platform.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ -#pragma once - -#include "hip_internal.hpp" -#include "hip_fatbin.hpp" -#include "device/device.hpp" -#include "hip_code_object.hpp" - -namespace hip_impl { - -hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor( - int* maxBlocksPerCU, int* numBlocksPerGrid, int* bestBlockSize, - const amd::Device& device, hipFunction_t func, int blockSize, - size_t dynamicSMemSize, bool bCalcPotentialBlkSz); -} /* namespace hip_impl*/ - -class PlatformState { - amd::Monitor lock_{"Guards PlatformState globals", true}; - - /* Singleton object */ - static PlatformState* platform_; - PlatformState() {} - ~PlatformState() {} - -public: - void init(); - - //Dynamic Code Objects functions - hipError_t loadModule(hipModule_t* module, const char* fname, const void* image = nullptr); - hipError_t unloadModule(hipModule_t hmod); - - hipError_t getDynFunc(hipFunction_t *hfunc, hipModule_t hmod, const char* func_name); - hipError_t getDynGlobalVar(const char* hostVar, hipModule_t hmod, hipDeviceptr_t* dev_ptr, - size_t* size_ptr); - hipError_t getDynTexRef(const char* hostVar, hipModule_t hmod, textureReference** texRef); - - hipError_t registerTexRef(textureReference* texRef, hipModule_t hmod, std::string name); - hipError_t getDynTexGlobalVar(textureReference* texRef, hipDeviceptr_t* dev_ptr, - size_t* size_ptr); - - /* Singleton instance */ - static PlatformState& instance() { - if (platform_ == nullptr) { - // __hipRegisterFatBinary() will call this when app starts, thus - // there is no multiple entry issue here. - platform_ = new PlatformState(); - } - return *platform_; - } - - //Static Code Objects functions - hip::FatBinaryInfo** addFatBinary(const void* data); - hipError_t removeFatBinary(hip::FatBinaryInfo** module); - hipError_t digestFatBinary(const void* data, hip::FatBinaryInfo*& programs); - - hipError_t registerStatFunction(const void* hostFunction, hip::Function* func); - hipError_t registerStatGlobalVar(const void* hostVar, hip::Var* var); - hipError_t registerStatManagedVar(hip::Var* var); - - - hipError_t getStatFunc(hipFunction_t* hfunc, const void* hostFunction, int deviceId); - hipError_t getStatFuncAttr(hipFuncAttributes* func_attr, const void* hostFunction, int deviceId); - hipError_t getStatGlobalVar(const void* hostVar, int deviceId, hipDeviceptr_t* dev_ptr, - size_t* size_ptr); - - hipError_t initStatManagedVarDevicePtr(int deviceId); - - //Exec Functions - void setupArgument(const void *arg, size_t size, size_t offset); - void configureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream); - void popExec(ihipExec_t& exec); - -private: - //Dynamic Code Object map, keyin module to get the corresponding object - std::unordered_map dynCO_map_; - hip::StatCO statCO_; //Static Code object var - bool initialized_{false}; - std::unordered_map> texRef_map_; -}; diff --git a/src/hipamd/src/hip_prof_api.h b/src/hipamd/src/hip_prof_api.h deleted file mode 100644 index 8773f9ac1f..0000000000 --- a/src/hipamd/src/hip_prof_api.h +++ /dev/null @@ -1,270 +0,0 @@ -/* Copyright (c) 2019-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef HIP_SRC_HIP_PROF_API_H -#define HIP_SRC_HIP_PROF_API_H - -#include -#include -#include - -#if USE_PROF_API -#include "hip/amd_detail/hip_prof_str.h" -#include "platform/prof_protocol.h" - -// HIP API callbacks spawner object macro -#define HIP_CB_SPAWNER_OBJECT(CB_ID) \ - api_callbacks_spawner_t __api_tracer; \ - { \ - hip_api_data_t* api_data = __api_tracer.get_api_data_ptr(); \ - if (api_data != NULL) { \ - hip_api_data_t& api_data_ref = *api_data; \ - INIT_CB_ARGS_DATA(CB_ID, api_data_ref); \ - __api_tracer.call(); \ - } \ - } - -static const uint32_t HIP_DOMAIN_ID = ACTIVITY_DOMAIN_HIP_API; -typedef activity_record_t hip_api_record_t; -typedef activity_rtapi_callback_t hip_api_callback_t; -typedef activity_sync_callback_t hip_act_callback_t; - -class api_callbacks_table_t { - public: - typedef std::mutex mutex_t; - - typedef hip_api_record_t record_t; - typedef hip_api_callback_t fun_t; - typedef hip_act_callback_t act_t; - - // HIP API callbacks table - struct hip_cb_table_entry_t { - volatile std::atomic sync; - volatile std::atomic sem; - act_t act; - void* a_arg; - fun_t fun; - void* arg; - }; - - struct hip_cb_table_t { - hip_cb_table_entry_t arr[HIP_API_ID_NUMBER]; - }; - - api_callbacks_table_t() { - memset(&callbacks_table_, 0, sizeof(callbacks_table_)); - } - - bool set_activity(uint32_t id, act_t fun, void* arg) { - std::lock_guard lock(mutex_); - bool ret = true; - - if (id < HIP_API_ID_NUMBER) { - cb_sync(id); - /* - 'fun != nullptr' indicates it is activity register call, - increment should happen only once but client is free to call - register CB multiple times for same API id hence the check - - 'fun == nullptr' indicates it is de-register call and - decrement should happen only once hence the check - */ - if (fun != nullptr) { - if (callbacks_table_.arr[id].act == nullptr) { - enabled_api_count_++; - } - } else { - if (callbacks_table_.arr[id].act != nullptr) { - enabled_api_count_--; - } - } - if (enabled_api_count_ > 0) { - amd::IS_PROFILER_ON = true; - } else { - amd::IS_PROFILER_ON = false; - } - callbacks_table_.arr[id].act = fun; - callbacks_table_.arr[id].a_arg = arg; - cb_release(id); - } else { - ret = false; - } - - return ret; - } - - bool set_callback(uint32_t id, fun_t fun, void* arg) { - std::lock_guard lock(mutex_); - bool ret = true; - - if (id < HIP_API_ID_NUMBER) { - cb_sync(id); - callbacks_table_.arr[id].fun = fun; - callbacks_table_.arr[id].arg = arg; - cb_release(id); - } else { - ret = false; - } - - return ret; - } - - void set_enabled(const bool& enabled) { - amd::IS_PROFILER_ON = enabled; - } - - inline hip_cb_table_entry_t& entry(const uint32_t& id) { - return callbacks_table_.arr[id]; - } - - inline void sem_sync(const uint32_t& id) { - sem_increment(id); - if (entry(id).sync.load() == true) sync_wait(id); - } - - inline void sem_release(const uint32_t& id) { - sem_decrement(id); - } - - inline bool is_enabled() const { - return amd::IS_PROFILER_ON; - } - - private: - inline void cb_sync(const uint32_t& id) { - entry(id).sync.store(true); - while (entry(id).sem.load() != 0) {} - } - - inline void cb_release(const uint32_t& id) { - entry(id).sync.store(false); - } - - inline void sem_increment(const uint32_t& id) { - const uint32_t prev = entry(id).sem.fetch_add(1); - if (prev == UINT32_MAX) { - std::cerr << "sem overflow id = " << id << std::endl << std::flush; - abort(); - } - } - - inline void sem_decrement(const uint32_t& id) { - const uint32_t prev = entry(id).sem.fetch_sub(1); - if (prev == 0) { - std::cerr << "sem corrupted id = " << id << std::endl << std::flush; - abort(); - } - } - - void sync_wait(const uint32_t& id) { - sem_decrement(id); - while (entry(id).sync.load() == true) {} - sem_increment(id); - } - - mutex_t mutex_; - hip_cb_table_t callbacks_table_; - uint32_t enabled_api_count_; -}; - -extern api_callbacks_table_t callbacks_table; - -template -class api_callbacks_spawner_t { - public: - api_callbacks_spawner_t() : - api_data_(NULL) - { - if (!is_enabled()) return; - - if (cid_ >= HIP_API_ID_NUMBER) { - fprintf(stderr, "HIP %s bad id %d\n", __FUNCTION__, cid_); - abort(); - } - callbacks_table.sem_sync(cid_); - - hip_act_callback_t act = entry(cid_).act; - if (act != NULL) api_data_ = (hip_api_data_t*) act(cid_, NULL, NULL, NULL); - } - - void call() { - hip_api_callback_t fun = entry(cid_).fun; - void* arg = entry(cid_).arg; - if (fun != NULL) { - fun(HIP_DOMAIN_ID, cid_, api_data_, arg); - api_data_->phase = ACTIVITY_API_PHASE_EXIT; - } - } - - ~api_callbacks_spawner_t() { - if (!is_enabled()) return; - - if (api_data_ != NULL) { - hip_api_callback_t fun = entry(cid_).fun; - void* arg = entry(cid_).arg; - hip_act_callback_t act = entry(cid_).act; - void* a_arg = entry(cid_).a_arg; - if (fun != NULL) fun(HIP_DOMAIN_ID, cid_, api_data_, arg); - if (act != NULL) act(cid_, NULL, NULL, a_arg); - } - - callbacks_table.sem_release(cid_); - } - - hip_api_data_t* get_api_data_ptr() { - return api_data_; - } - - bool is_enabled() const { - return callbacks_table.is_enabled(); - } - - private: - inline api_callbacks_table_t::hip_cb_table_entry_t& entry(const uint32_t& id) { - return callbacks_table.entry(id); - } - - hip_api_data_t* api_data_; -}; - -template <> -class api_callbacks_spawner_t { - public: - api_callbacks_spawner_t() {} - void call() {} - hip_api_data_t* get_api_data_ptr() { return NULL; } - bool is_enabled() const { return false; } -}; - -#else - -#define HIP_CB_SPAWNER_OBJECT(x) do {} while(0) - -class api_callbacks_table_t { - public: - typedef void* act_t; - typedef void* fun_t; - bool set_activity(uint32_t id, act_t fun, void* arg) { return false; } - bool set_callback(uint32_t id, fun_t fun, void* arg) { return false; } -}; - -#endif - -#endif // HIP_SRC_HIP_PROF_API_H diff --git a/src/hipamd/src/hip_prof_gen.py b/src/hipamd/src/hip_prof_gen.py deleted file mode 100755 index aa44a70bd2..0000000000 --- a/src/hipamd/src/hip_prof_gen.py +++ /dev/null @@ -1,673 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2019-present Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -import os, sys, re - -PROF_HEADER = "hip_prof_str.h" -OUTPUT = PROF_HEADER -REC_MAX_LEN = 1024 - -# Recursive sources processing -recursive_mode = 0 -# HIP_INIT_API macro patching -hip_patch_mode = 0 -# API matching types check -types_check_mode = 0 -# Private API check -private_check_mode = 0 - -# Messages and errors controll -verbose = 0 -errexit = 0 -inp_file = 'none' -line_num = -1 - -# Verbose message -def message(msg): - if verbose: sys.stdout.write(msg + '\n') - -# Fatal error termination -def error(msg): - if line_num != -1: - msg += ", file '" + inp_file + "', line (" + str(line_num) + ")" - if errexit: - msg = " Error: " + msg - else: - msg = " Warning: " + msg - - sys.stdout.write(msg + '\n') - sys.stderr.write(sys.argv[0] + msg +'\n') - -def fatal(msg): - error(msg) - sys.exit(1) - -############################################################# -# Normalizing API name -def filtr_api_name(name): - name = re.sub(r'\s*$', r'', name); - return name - -def filtr_api_decl(record): - record = re.sub("\s__dparm\([^\)]*\)", r'', record); - record = re.sub("\(void\*\)", r'', record); - return record - -# Normalizing API arguments -def filtr_api_args(args_str): - args_str = re.sub(r'^\s*', r'', args_str); - args_str = re.sub(r'\s*$', r'', args_str); - args_str = re.sub(r'\s*,\s*', r',', args_str); - args_str = re.sub(r'\s+', r' ', args_str); - args_str = re.sub(r'\s*(\*+)\s*', r'\1 ', args_str); - args_str = re.sub(r'(enum|struct) ', '', args_str); - return args_str - -# Normalizing types -def norm_api_types(type_str): - type_str = re.sub(r'uint32_t', r'unsigned int', type_str) - type_str = re.sub(r'^unsigned$', r'unsigned int', type_str) - return type_str - -# Creating a list of arguments [(type, name), ...] -def list_api_args(args_str): - args_str = filtr_api_args(args_str) - args_list = [] - if args_str != '': - for arg_pair in args_str.split(','): - if arg_pair == 'void': continue - arg_pair = re.sub(r'\s*=\s*\S+$','', arg_pair); - m = re.match("^(.*)\s(\S+)$", arg_pair); - if m: - arg_type = norm_api_types(m.group(1)) - arg_name = m.group(2) - args_list.append((arg_type, arg_name)) - else: - fatal("bad args: args_str: '" + args_str + "' arg_pair: '" + arg_pair + "'") - return args_list; - -# Creating arguments string "type0, type1, ..." -def filtr_api_types(args_str): - args_list = list_api_args(args_str) - types_str = '' - for arg_tuple in args_list: - types_str += arg_tuple[0] + ', ' - return types_str - -# Creating options list [opt0, opt1, ...] -def filtr_api_opts(args_str): - args_list = list_api_args(args_str) - opts_list = [] - for arg_tuple in args_list: - opts_list.append(arg_tuple[1]) - return opts_list - -# Checking for pointer non-void arg type -def pointer_ck(arg_type): - ptr_type = '' - m = re.match(r'(.*)\*$', arg_type) - if m: - ptr_type = m.group(1) - ptr_type = re.sub(r'const ', '', ptr_type) - if ptr_type == 'void': ptr_type = '' - return ptr_type -############################################################# -# Parsing API header -# hipError_t hipSetupArgument(const void* arg, size_t size, size_t offset); -def parse_api(inp_file_p, out): - global inp_file - global line_num - inp_file = inp_file_p - - beg_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\("); - api_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\(([^\)]*)\)"); - end_pattern = re.compile("Texture"); - hidden_pattern = re.compile(r'__attribute__\(\(visibility\("hidden"\)\)\)') - nms_open_pattern = re.compile(r'namespace hip_impl {') - nms_close_pattern = re.compile(r'}') - - inp = open(inp_file, 'r') - - found = 0 - hidden = 0 - nms_level = 0; - record = "" - line_num = -1 - - for line in inp.readlines(): - record += re.sub(r'^\s+', r' ', line[:-1]) - line_num += 1 - - if len(record) > REC_MAX_LEN: - fatal("bad record \"" + record + "\"") - - m = beg_pattern.match(line) - if m: - name = m.group(2) - if hidden != 0: - message("api: " + name + " - hidden") - elif nms_level != 0: - message("api: " + name + " - hip_impl") - else: - message("api: " + name) - found = 1 - - if found != 0: - record = re.sub("\s__dparm\([^\)]*\)", '', record); - m = api_pattern.match(record) - if m: - found = 0 - if end_pattern.search(record): break - api_name = filtr_api_name(m.group(2)) - api_args = m.group(3) - if not api_name in out: - out[api_name] = api_args - else: continue - - hidden = 0 - if hidden_pattern.match(line): hidden = 1 - - if nms_open_pattern.match(line): nms_level += 1 - if (nms_level > 0) and nms_close_pattern.match(line): nms_level -= 1 - if nms_level < 0: - fatal("nms level < 0") - - record = "" - - inp.close() - line_num = -1 -############################################################# -# Parsing API implementation -# hipError_t hipSetupArgument(const void* arg, size_t size, size_t offset) { -# HIP_INIT_API(hipSetupArgument, arg, size, offset); -# inp_file - input implementation source file -# api_map - input public API map [] => -# out - output map [] => [opt0, opt1, ...] -def parse_content(inp_file_p, api_map, out): - global hip_patch_mode - global types_check_mode - global private_check_mode - global inp_file - global line_num - inp_file = inp_file_p - - # API method begin pattern - beg_pattern = re.compile("^(hipError_t|const char\s*\*)\s+[^\(]+\("); - # API declaration pattern - decl_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\(([^\)]*)\)\s*;"); - # API definition pattern - api_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\(([^\)]*)\)\s*{"); - # API init macro pattern - init_pattern = re.compile("(^\s*HIP_INIT_API\s*)\((([^,]+)(,.*|)|)(\);|,)\s*$"); - - # Open input file - inp = open(inp_file, 'r') - - # API name - api_name = "" - # Valid public API found flag - api_valid = 0 - # API overload (parameters mismatch) - api_overload = 0 - - # Input file patched content - content = '' - # Sub content for found API defiition - sub_content = '' - # Current record, accumulating several API definition related lines - record = '' - # Current input file line number - line_num = -1 - # API beginning found flag - found = 0 - - # Reading input file - for line in inp.readlines(): - # Accumulating record - record += re.sub(r'^\s+', r' ', line[:-1]) - line_num += 1 - - if len(record) > REC_MAX_LEN: - fatal("bad record \"" + record + "\"") - break; - - # Looking for API begin - if found == 0: - record = re.sub(r'\s*extern\s+"C"\s+', r'', record); - if beg_pattern.match(record): - found = 1 - record = filtr_api_decl(record) - - # Matching API declaration - if found == 1: - if decl_pattern.match(record): - found = 0 - - # Matching API definition - if found == 1: - m = api_pattern.match(record) - # Checking if complete API matched - if m: - found = 2 - api_valid = 0 - api_overload = 0 - api_name = filtr_api_name(m.group(2)) - # Checking if API name is in the API map - if (private_check_mode == 0) or (api_name in api_map): - if not api_name in api_map: api_map[api_name] = '' - # Getting API arguments - api_args = m.group(3) - # Getting etalon arguments from the API map - eta_args = api_map[api_name] - if eta_args == '': - eta_args = api_args - api_map[api_name] = eta_args - # Normalizing API arguments - api_types = filtr_api_types(api_args) - # Normalizing etalon arguments - eta_types = filtr_api_types(eta_args) - if (api_types == eta_types) or ((types_check_mode == 0) and (not api_name in out)): - # API is already found and not is mismatched - if (api_name in out): - fatal("API redefined \"" + api_name + "\", record \"" + record + "\"") - # Set valid public API found flag - api_valid = 1 - # Set output API map with API arguments list - out[api_name] = filtr_api_opts(api_args) - # Register missmatched API methods - else: - api_overload = 1 - # Warning about mismatched API, possible non public overloaded version - api_diff = '\t\t' + inp_file + " line(" + str(line_num) + ")\n\t\tapi: " + api_types + "\n\t\teta: " + eta_types - message("\t" + api_name + ' args mismatch:\n' + api_diff + '\n') - - # API found action - if found == 2: - if hip_patch_mode != 0: - # Looking for INIT macro - m = init_pattern.match(line) - if m: - init_name = api_name - if api_overload == 1: init_name = 'NONE' - init_args = m.group(4) - line = m.group(1) + '(' + init_name + init_args + m.group(5) + '\n' - - m = init_pattern.match(line) - if m: - found = 0 - if api_valid == 1: message("\t" + api_name) - # Ignore if it is initialized as NONE - init_name = m.group(3) - if init_name != 'NONE': - # Check if init name matching API name - if init_name != api_name: - fatal("init name mismatch: '" + init_name + "' <> '" + api_name + "'") - # Registering dummy API for non public API if the name in INIT is not NONE - if api_valid == 0: - # If init name is not in public API map then it is private API - # else it was not identified and will be checked on finish - if not init_name in api_map: - if init_name in out: - fatal("API reinit \"" + api_name + "\", record \"" + record + "\"") - out[init_name] = [] - elif re.search('}', line): - found = 0 - # Expect INIT macro for valid public API - # Removing and registering non-conformant APIs with missing HIP_INIT macro - if api_valid == 1: - if api_name in out: - del out[api_name] - del api_map[api_name] - # Registering non-conformant APIs - out['.' + api_name] = 1 - else: - fatal("API is not in out \"" + api_name + "\", record \"" + record + "\"") - - if found != 1: record = "" - content += line - - inp.close() - line_num = -1 - - if len(out) != 0: - return content - else: - return '' - -# src path walk -def parse_src(api_map, src_path, src_patt, out): - global recursive_mode - - pattern = re.compile(src_patt) - src_path = re.sub(r'\s', '', src_path) - for src_dir in src_path.split(':'): - message("Parsing " + src_dir + " for '" + src_patt + "'") - for root, dirs, files in os.walk(src_dir): - for fnm in files: - if pattern.search(fnm): - file = root + '/' + fnm - message(file) - content = parse_content(file, api_map, out); - if (hip_patch_mode != 0) and (content != ''): - f = open(file, 'w') - f.write(content) - f.close() - if recursive_mode == 0: break -############################################################# -# Generating profiling primitives header -# api_map - public API map [] => [(type, name), ...] -# opts_map - opts map [] => [opt0, opt1, ...] -def generate_prof_header(f, api_map, opts_map): - # Private API list - priv_lst = [] - - f.write('// automatically generated sources\n') - f.write('#ifndef _HIP_PROF_STR_H\n'); - f.write('#define _HIP_PROF_STR_H\n'); - f.write('#define HIP_PROF_VER 1\n') - - # Generating dummy macro for non-public API - f.write('\n// Dummy API primitives\n') - f.write('#define INIT_NONE_CB_ARGS_DATA(cb_data) {};\n') - for name in opts_map: - if not name in api_map: - opts_lst = opts_map[name] - if len(opts_lst) != 0: - fatal("bad dummy API \"" + name + "\", args: " + str(opts_lst)) - f.write('#define INIT_'+ name + '_CB_ARGS_DATA(cb_data) {};\n') - priv_lst.append(name) - - for name in priv_lst: - message("Private: " + name) - - # Generating the callbacks ID enumaration - f.write('\n// HIP API callbacks ID enumaration\n') - f.write('enum hip_api_id_t {\n') - cb_id = 0 - for name in api_map.keys(): - f.write(' HIP_API_ID_' + name + ' = ' + str(cb_id) + ',\n') - cb_id += 1 - f.write(' HIP_API_ID_NUMBER = ' + str(cb_id) + ',\n') - f.write('\n') - f.write(' HIP_API_ID_NONE = HIP_API_ID_NUMBER,\n') - for name in priv_lst: - f.write(' HIP_API_ID_' + name + ' = HIP_API_ID_NUMBER,\n') - f.write('};\n') - - # Generating the method to return API name by ID - f.write('\n// Return HIP API string by given ID\n') - f.write('static inline const char* hip_api_name(const uint32_t id) {\n') - f.write(' switch(id) {\n') - for name in api_map.keys(): - f.write(' case HIP_API_ID_' + name + ': return "' + name + '";\n') - f.write(' };\n') - f.write(' return "unknown";\n') - f.write('};\n') - - # Generating the method for querying API ID by name - f.write('\n') - f.write('#include \n'); - f.write('// Return HIP API ID by given name\n') - f.write('static inline uint32_t hipApiIdByName(const char* name) {\n') - for name, args in api_map.items(): - f.write(' if (strcmp("' + name + '", name) == 0) return HIP_API_ID_' + name + ';\n') - f.write(' return HIP_API_ID_NUMBER;\n') - f.write('}\n') - - # Generating the callbacks data structure - f.write('\n// HIP API callbacks data structure\n') - f.write( - 'typedef struct hip_api_data_s {\n' + - ' uint64_t correlation_id;\n' + - ' uint32_t phase;\n' + - ' union {\n' - ) - for name, args in api_map.items(): - if len(args) != 0: - f.write(' struct {\n') - for arg_tuple in args: - arg_type = arg_tuple[0] - ptr_type = pointer_ck(arg_type) - arg_name = arg_tuple[1] - # Checking for enum type - if arg_type == "hipLimit_t": arg_type = 'enum ' + arg_type - # Structuer field code - f.write(' ' + arg_type + ' ' + arg_name + ';\n') - if ptr_type != '': - f.write(' ' + ptr_type + ' ' + arg_name + '__val;\n') - f.write(' } ' + name + ';\n') - f.write( - ' } args;\n' + - '} hip_api_data_t;\n' - ) - - # Generating the callbacks args data filling macros - f.write('\n// HIP API callbacks args data filling macros\n') - for name, args in api_map.items(): - f.write('// ' + name + str(args) + '\n') - f.write('#define INIT_' + name + '_CB_ARGS_DATA(cb_data) { \\\n') - if name in opts_map: - opts_list = opts_map[name] - if len(args) != len(opts_list): - fatal("\"" + name + "\" API args and opts mismatch, args: " + str(args) + ", opts: " + str(opts_list)) - # API args iterating: - # type is args[][0] - # name is args[][1] - for ind in range(0, len(args)): - arg_tuple = args[ind] - arg_type = arg_tuple[0] - ptr_type = pointer_ck(arg_type) - fld_name = arg_tuple[1] - opt_name = opts_list[ind] - if arg_type == "const char*": - f.write(' cb_data.args.' + name + '.' + fld_name + ' = (' + opt_name + ') ? strdup(' + opt_name + ') : NULL; \\\n') - else: - f.write(' cb_data.args.' + name + '.' + fld_name + ' = (' + arg_type + ')' + opt_name + '; \\\n') - f.write('};\n') - f.write('#define INIT_CB_ARGS_DATA(cb_id, cb_data) INIT_##cb_id##_CB_ARGS_DATA(cb_data)\n') - - f.write('#if HIP_PROF_HIP_API_STRING\n') - # Generating the method for the API args filling - f.write('\n') - f.write('// HIP API args filling method\n') - f.write('static inline void hipApiArgsInit(hip_api_id_t id, hip_api_data_t* data) {\n') - f.write(' switch (id) {\n') - for name, args in api_map.items(): - f.write('// ' + name + str(args) + '\n') - f.write(' case HIP_API_ID_' + name + ':\n') - for ind in range(0, len(args)): - arg_tuple = args[ind] - arg_type = arg_tuple[0] - ptr_type = pointer_ck(arg_type) - fld_name = arg_tuple[1] - var_name = 'data->args.' + name + '.' + fld_name - if arg_type == "char*": - f.write(' ' + var_name + ' = (' + var_name + ') ? strdup(' + var_name + ') : NULL;\n') - else: - if ptr_type != '': - f.write(' if (' + var_name + ') ' + var_name + '__val = *(' + var_name + ');\n') - f.write(' break;\n') - f.write(' default: break;\n') - f.write(' };\n') - f.write('}\n') - - # Generating the method for the API string, name and parameters - f.write('\n') - f.write('#include \n'); - f.write('#include \n'); - f.write('// HIP API string method, method name and parameters\n') - f.write('static inline const char* hipApiString(hip_api_id_t id, const hip_api_data_t* data) {\n') - f.write(' std::ostringstream oss;\n') - f.write(' switch (id) {\n') - for name, args in api_map.items(): - f.write(' case HIP_API_ID_' + name + ':\n') - f.write(' oss << "' + name + '(";\n') - for ind in range(0, len(args)): - arg_tuple = args[ind] - arg_type = arg_tuple[0] - ptr_type = pointer_ck(arg_type) - arg_name = arg_tuple[1] - var_name = 'data->args.' + name + '.' + arg_name - delim = '' if ind == 0 else ', '; - oss_stream = 'oss << "' + delim + arg_name + '=' - line_shift = ' ' - f.write(line_shift) - if ptr_type != '': - f.write('if (' + var_name + ' == NULL) ' + oss_stream + 'NULL";\n' + line_shift + 'else ') - if pointer_ck(ptr_type) != '': - f.write(oss_stream + '" << (void*)' + var_name + '__val' + ';\n') - else: - f.write(oss_stream + '" << ' + var_name + '__val' + ';\n') - else: - f.write(oss_stream + '" << ' + var_name + ';\n') - f.write(' oss << ")";\n') - f.write(' break;\n') - f.write(' default: oss << "unknown";\n') - f.write(' };\n') - f.write(' return strdup(oss.str().c_str());\n') - f.write('}\n') - f.write('#endif // HIP_PROF_HIP_API_STRING\n') - - f.write('#endif // _HIP_PROF_STR_H\n'); - -############################################################# -# main -while len(sys.argv) > 1: - if not re.match(r'-', sys.argv[1]): break - - if (sys.argv[1] == '-v'): - verbose = 1 - sys.argv.pop(1) - - if (sys.argv[1] == '-r'): - recursive_mode = 1 - sys.argv.pop(1) - - if (sys.argv[1] == '-t'): - types_check_mode = 1 - sys.argv.pop(1) - - if (sys.argv[1] == '--priv'): - private_check_mode = 1 - sys.argv.pop(1) - - if (sys.argv[1] == '-e'): - errexit = 1 - sys.argv.pop(1) - - if (sys.argv[1] == '-p'): - hip_patch_mode = 1 - sys.argv.pop(1) - -# Usage -if (len(sys.argv) < 3): - fatal ("Usage: " + sys.argv[0] + " [-v] []\n" + - " -v - verbose messages\n" + - " -r - process source directory recursively\n" + - " -t - API types matching check\n" + - " --priv - private API check\n" + - " -e - on error exit mode\n" + - " -p - HIP_INIT_API macro patching mode\n" + - "\n" + - " Example:\n" + - " $ " + sys.argv[0] + " -v -p -t --priv ./api/hip/include/hip/amd_detail/hip_runtime_api.h ./api/hip ./api/hip/include/hip/amd_detail/hip_prof_str.h"); - -# API header file given as an argument -src_pat = "\.cpp$" -api_hfile = sys.argv[1] -if not os.path.isfile(api_hfile): - fatal("input file '" + api_hfile + "' not found") - -# Srcs directory given as an argument -src_dir = sys.argv[2] -if not os.path.isdir(src_dir): - fatal("src directory " + src_dir + "' not found") - -if len(sys.argv) > 3: OUTPUT = sys.argv[3] - -# API declaration map -api_map = { - 'hipSetupArgument': '', - 'hipMalloc3DArray': '', - 'hipFuncGetAttribute': '', - 'hipMemset3DAsync': '', - 'hipKernelNameRef': '', - 'hipStreamGetPriority': '', - 'hipLaunchByPtr': '', - 'hipFreeHost': '', - 'hipGetErrorName': '', - 'hipMemcpy3DAsync': '', - 'hipMemcpyParam2DAsync': '', - 'hipArray3DCreate': '', - 'hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags': '', - 'hipOccupancyMaxPotentialBlockSize': '', - 'hipMallocManaged': '', - 'hipOccupancyMaxActiveBlocksPerMultiprocessor': '', - 'hipGetErrorString': '', - 'hipMallocHost': '', - 'hipModuleLoadDataEx': '', - 'hipGetDeviceProperties': '', - 'hipConfigureCall': '', - 'hipHccModuleLaunchKernel': '', - 'hipExtModuleLaunchKernel': '', -} -# API options map -opts_map = {} - -# Parsing API header -parse_api(api_hfile, api_map) - -# Parsing sources -parse_src(api_map, src_dir, src_pat, opts_map) - -# Checking for non-conformant APIs with missing HIP_INIT macro -for name in list(opts_map.keys()): - m = re.match(r'\.(\S*)', name) - if m: - message("Init missing: " + m.group(1)) - del opts_map[name] - -# Converting api map to map of lists -# Checking for not found APIs -not_found = 0 -if len(opts_map) != 0: - for name in api_map.keys(): - args_str = api_map[name]; - api_map[name] = list_api_args(args_str) - if not name in opts_map: - error("implementation not found: " + name) - not_found += 1 -if not_found != 0: - error(str(not_found) + " API calls missing in interception layer") - -# The output subdirectory seems to exist or not depending on the -# version of cmake. -output_dir = os.path.dirname(OUTPUT) -if not os.path.exists(output_dir): - os.makedirs(output_dir) - -# Generating output header file -with open(OUTPUT, 'w') as f: - generate_prof_header(f, api_map, opts_map) - -# Successfull exit -sys.exit(0) diff --git a/src/hipamd/src/hip_profile.cpp b/src/hipamd/src/hip_profile.cpp deleted file mode 100644 index 3422f428ea..0000000000 --- a/src/hipamd/src/hip_profile.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include - -#include "hip_internal.hpp" - -hipError_t hipProfilerStart() { - HIP_INIT_API(hipProfilerStart); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - - -hipError_t hipProfilerStop() { - HIP_INIT_API(hipProfilerStop); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} diff --git a/src/hipamd/src/hip_rtc.cpp b/src/hipamd/src/hip_rtc.cpp deleted file mode 100755 index 66b6ad6e43..0000000000 --- a/src/hipamd/src/hip_rtc.cpp +++ /dev/null @@ -1,419 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include "hiprtc_internal.hpp" -#include -#include "platform/program.hpp" - -#ifdef __HIP_ENABLE_PCH -extern const char __hip_pch[]; -extern unsigned __hip_pch_size; -void __hipGetPCH(const char** pch, unsigned int *size) { - *pch = __hip_pch; - *size = __hip_pch_size; -} -#endif - -namespace hiprtc { -thread_local hiprtcResult g_lastRtcError = HIPRTC_SUCCESS; -} - -class ProgramState { - amd::Monitor lock_; -private: - static ProgramState* programState_; - - ProgramState() : lock_("Guards program state") {} - ~ProgramState() {} -public: - std::unordered_map, std::vector>> progHeaders_; - - std::map> nameExpresssion_; - - static ProgramState& instance(); - uint32_t addNameExpression(const char* name_expression); - char* getLoweredName(const char* name_expression); -}; - -ProgramState* ProgramState::programState_ = nullptr; - -ProgramState& ProgramState::instance() { - if (programState_ == nullptr) { - programState_ = new ProgramState; - } - return *programState_; -} - -uint32_t ProgramState::addNameExpression(const char* name_expression) { - amd::ScopedLock lock(lock_); - - // Strip clean of any '(' or ')' or '&' - std::string strippedName(name_expression); - if (strippedName.back() == ')') { - strippedName.pop_back(); - strippedName.erase(0, strippedName.find('(')); - } - if (strippedName.front() == '&') { - strippedName.erase(0, 1); - } - auto it = nameExpresssion_.find(name_expression); - if (it == nameExpresssion_.end()) { - nameExpresssion_.insert(std::pair> - (name_expression, std::make_pair(strippedName,""))); - } - return nameExpresssion_.size(); -} - -char* demangle(const char* loweredName) { - if (!loweredName) { - return nullptr; - } -#if __linux__ - int status = 0; - char* demangledName = DEMANGLE(loweredName, nullptr, nullptr, &status); - if (status != 0) { - LogPrintfError("Cannot demangle loweredName: %s \n", loweredName); - return nullptr; - } -#elif defined(_WIN32) - char* demangledName = (char*)malloc(UNDECORATED_SIZE); - - if (!UnDecorateSymbolName(loweredName, demangledName, - UNDECORATED_SIZE/ sizeof(*demangledName), UNDNAME_COMPLETE)) - { - free(demangledName); - LogPrintfError("Cannot undecorate loweredName: %s demangledName: %s \n", - loweredName, demangledName); - return nullptr; - } -#else -#error "Only Linux and Windows are supported" -#endif // __linux__ - return demangledName; -} - -static std::string handleMangledName(std::string name) { - std::string loweredName; - char* demangled = demangle(name.c_str()); - loweredName.assign(demangled == nullptr ? std::string() : demangled); - free(demangled); - - if (loweredName.empty()) { - return name; - } - - if (loweredName.find(".kd") != std::string::npos) { - return {}; - } - - if (loweredName.find("void ") == 0) { - loweredName.erase(0, strlen("void ")); - } - - auto dx{loweredName.find_first_of("(<")}; - - if (dx == std::string::npos) { - return loweredName; - } - - if (loweredName[dx] == '<') { - uint32_t count = 1; - do { - ++dx; - count += (loweredName[dx] == '<') ? 1 : ((loweredName[dx] == '>') ? -1 : 0); - } while (count); - - loweredName.erase(++dx); - } else { - loweredName.erase(dx); - } - - return loweredName; -} - -static std::string getValueOf(const std::string& option) { - std::string res; - auto f = std::find(option.begin(), option.end(), '='); - if (f != option.end()) res = std::string(f + 1, option.end()); - return res; -} - -static void transformOptions(std::vector& options, amd::Program* program) { - std::vector t_option; - for (auto& i : options) { -#ifdef __HIP_ENABLE_PCH - // Use precompiled header for hip - if (i == "-hip-pch") { - const char* pch = nullptr; - unsigned int pch_size = 0; - __hipGetPCH(&pch, &pch_size); - program->addPreCompiledHeader(std::string(pch, pch_size)); - i = "-nogpuinc"; - continue; - } -#endif - // Some rtc samples use --gpu-architecture - if (i.rfind("--gpu-architecture=", 0) == 0) { - auto val = getValueOf(i); - i = "--offload-arch=" + val; - continue; - } - } -} - -const char* hiprtcGetErrorString(hiprtcResult x) { - switch (x) { - case HIPRTC_SUCCESS: - return "HIPRTC_SUCCESS"; - case HIPRTC_ERROR_OUT_OF_MEMORY: - return "HIPRTC_ERROR_OUT_OF_MEMORY"; - case HIPRTC_ERROR_PROGRAM_CREATION_FAILURE: - return "HIPRTC_ERROR_PROGRAM_CREATION_FAILURE"; - case HIPRTC_ERROR_INVALID_INPUT: - return "HIPRTC_ERROR_INVALID_INPUT"; - case HIPRTC_ERROR_INVALID_PROGRAM: - return "HIPRTC_ERROR_INVALID_PROGRAM"; - case HIPRTC_ERROR_INVALID_OPTION: - return "HIPRTC_ERROR_INVALID_OPTION"; - case HIPRTC_ERROR_COMPILATION: - return "HIPRTC_ERROR_COMPILATION"; - case HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE: - return "HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE"; - case HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION: - return "HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION"; - case HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION: - return "HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION"; - case HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID: - return "HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID"; - case HIPRTC_ERROR_INTERNAL_ERROR: - return "HIPRTC_ERROR_INTERNAL_ERROR"; - default: - LogPrintfError("Invalid HIPRTC error code: %d \n", x); - return nullptr; - }; - - ShouldNotReachHere(); - - return nullptr; -} - -hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, const char* src, const char* name, - int numHeaders, const char** headers, const char** headerNames) { - HIPRTC_INIT_API(prog, src, name, numHeaders, headers, headerNames); - - if (prog == nullptr) { - HIPRTC_RETURN(HIPRTC_ERROR_INVALID_PROGRAM); - } - if (numHeaders < 0) { - HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT); - } - if (numHeaders && (headers == nullptr || headerNames == nullptr)) { - HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT); - } - - amd::Program* program = new amd::Program(*hip::getCurrentDevice()->asContext(), src, amd::Program::HIP, - numHeaders, headers, headerNames); - if (program == NULL) { - HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT); - } - - if (CL_SUCCESS != program->addDeviceProgram(*hip::getCurrentDevice()->devices()[0])) { - program->release(); - HIPRTC_RETURN(HIPRTC_ERROR_PROGRAM_CREATION_FAILURE); - } - - *prog = reinterpret_cast(as_cl(program)); - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, int numOptions, const char** options) { - - // FIXME[skudchad] Add headers to amd::Program::build and device::Program::build, - // pass the saved from ProgramState to amd::Program::build - HIPRTC_INIT_API(prog, numOptions, options); - - amd::Program* program = as_amd(reinterpret_cast(prog)); - - std::ostringstream ostrstr; - std::vector oarr(&options[0], &options[numOptions]); - - const std::string hipVerOpt{"--hip-version=" + std::to_string(HIP_VERSION_MAJOR) + '.' + - std::to_string(HIP_VERSION_MINOR) + '.' + - std::to_string(HIP_VERSION_PATCH)}; - const std::string hipVerMajor{"-DHIP_VERSION_MAJOR=" + std::to_string(HIP_VERSION_MAJOR)}; - const std::string hipVerMinor{"-DHIP_VERSION_MINOR=" + std::to_string(HIP_VERSION_MINOR)}; - const std::string hipVerPatch{"-DHIP_VERSION_PATCH=" + std::to_string(HIP_VERSION_PATCH)}; - - oarr.push_back(hipVerOpt); - oarr.push_back(hipVerMajor); - oarr.push_back(hipVerMinor); - oarr.push_back(hipVerPatch); - - transformOptions(oarr, program); - std::copy(oarr.begin(), oarr.end(), std::ostream_iterator(ostrstr, " ")); - - std::vector devices{hip::getCurrentDevice()->devices()[0]}; - if (CL_SUCCESS != program->build(devices, ostrstr.str().c_str(), nullptr, nullptr)) { - HIPRTC_RETURN(HIPRTC_ERROR_COMPILATION); - } - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcAddNameExpression(hiprtcProgram prog, const char* name_expression) { - HIPRTC_INIT_API(prog, name_expression); - - if (name_expression == nullptr) { - HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT); - } - amd::Program* program = as_amd(reinterpret_cast(prog)); - - uint32_t id = ProgramState::instance().addNameExpression(name_expression); - - const auto var{"__hiprtc_" + std::to_string(id)}; - const auto code{"\nextern \"C\" constexpr auto " + var + " = " + name_expression + ';'}; - - program->appendToSource(code.c_str()); - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcGetLoweredName(hiprtcProgram prog, const char* name_expression, - const char** loweredName) { - HIPRTC_INIT_API(prog, name_expression, loweredName); - - if (name_expression == nullptr || loweredName == nullptr) { - HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT); - } - - amd::Program* program = as_amd(reinterpret_cast(prog)); - - device::Program* dev_program - = program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]); - - auto it = ProgramState::instance().nameExpresssion_.find(name_expression); - if (it == ProgramState::instance().nameExpresssion_.end()) { - return HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID; - } - - std::string strippedName = it->second.first; - std::vector mangledNames; - - if (!dev_program->getLoweredNames(&mangledNames)) { - HIPRTC_RETURN(HIPRTC_ERROR_COMPILATION); - } - - for (auto &name : mangledNames) { - std::string demangledName = handleMangledName(name); - if (demangledName == strippedName) { - it->second.second.assign(name); - } - } - - *loweredName = it->second.second.c_str(); - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcDestroyProgram(hiprtcProgram* prog) { - HIPRTC_INIT_API(prog); - - if (prog == NULL) { - HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT); - } - - // Release program. hiprtcProgram is a double pointer so free *prog - amd::Program* program = as_amd(reinterpret_cast(*prog)); - - program->release(); - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcGetCode(hiprtcProgram prog, char* binaryMem) { - HIPRTC_INIT_API(prog, binaryMem); - - - amd::Program* program = as_amd(reinterpret_cast(prog)); - const device::Program::binary_t& binary = - program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0])->binary(); - - ::memcpy(binaryMem, binary.first, binary.second); - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, size_t* binarySizeRet) { - - HIPRTC_INIT_API(prog, binarySizeRet); - - amd::Program* program = as_amd(reinterpret_cast(prog)); - - *binarySizeRet = - program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0])->binary().second; - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* dst) { - - HIPRTC_INIT_API(prog, dst); - amd::Program* program = as_amd(reinterpret_cast(prog)); - const device::Program* devProgram = - program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]); - - auto log = program->programLog() + devProgram->buildLog().c_str(); - - log.copy(dst, log.size()); - dst[log.size()] = '\0'; - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog, size_t* logSizeRet) { - - HIPRTC_INIT_API(prog, logSizeRet); - - amd::Program* program = as_amd(reinterpret_cast(prog)); - const device::Program* devProgram = - program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]); - - auto log = program->programLog() + devProgram->buildLog().c_str(); - - *logSizeRet = log.size() + 1; - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} - -hiprtcResult hiprtcVersion(int* major, int* minor) { - HIPRTC_INIT_API(major, minor); - - if (major == nullptr || minor == nullptr) { - HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT); - } - - *major = 9; - *minor = 0; - - HIPRTC_RETURN(HIPRTC_SUCCESS); -} diff --git a/src/hipamd/src/hip_stream.cpp b/src/hipamd/src/hip_stream.cpp deleted file mode 100755 index 801895cfd1..0000000000 --- a/src/hipamd/src/hip_stream.cpp +++ /dev/null @@ -1,598 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include "hip_internal.hpp" -#include "hip_event.hpp" -#include "thread/monitor.hpp" -#include "hip_prof_api.h" - -extern api_callbacks_table_t callbacks_table; - -static amd::Monitor streamSetLock{"Guards global stream set"}; -static std::unordered_set streamSet; -namespace hip { - -// ================================================================================================ -Stream::Stream(hip::Device* dev, Priority p, unsigned int f, bool null_stream, - const std::vector& cuMask, hipStreamCaptureStatus captureStatus) - : queue_(nullptr), - lock_("Stream Callback lock"), - device_(dev), - priority_(p), - flags_(f), - null_(null_stream), - cuMask_(cuMask), - captureStatus_(captureStatus) {} - -// ================================================================================================ -Stream::~Stream() { - if (queue_ != nullptr) { - amd::ScopedLock lock(streamSetLock); - streamSet.erase(this); - - queue_->release(); - queue_ = nullptr; - } -} - -hipError_t Stream::EndCapture() { - for (auto event : captureEvents_) { - hip::Event* e = reinterpret_cast(event); - e->EndCapture(); - } - for (auto stream : parallelCaptureStreams_) { - hip::Stream* s = reinterpret_cast(stream); - s->EndCapture(); - } - captureStatus_ = hipStreamCaptureStatusNone; - pCaptureGraph_ = nullptr; - originStream_ = false; - parentStream_ = nullptr; - lastCapturedNodes_.clear(); - parallelCaptureStreams_.clear(); - captureEvents_.clear(); - - return hipSuccess; -} -// ================================================================================================ -bool Stream::Create() { - // Enable queue profiling if a profiler is attached which sets the callback_table flag - // or if we force it with env var. This would enable time stamp collection for every - // command submitted to the stream(queue). - bool isProfilerAttached = callbacks_table.is_enabled(); - cl_command_queue_properties properties = (isProfilerAttached || HIP_FORCE_QUEUE_PROFILING) ? - CL_QUEUE_PROFILING_ENABLE : 0; - amd::CommandQueue::Priority p; - switch (priority_) { - case Priority::High: - p = amd::CommandQueue::Priority::High; - break; - case Priority::Low: - p = amd::CommandQueue::Priority::Low; - break; - case Priority::Normal: - default: - p = amd::CommandQueue::Priority::Normal; - break; - } - amd::HostQueue* queue = new amd::HostQueue(*device_->asContext(), *device_->devices()[0], - properties, amd::CommandQueue::RealTimeDisabled, - p, cuMask_); - - // Create a host queue - bool result = (queue != nullptr) ? queue->create() : false; - // Insert just created stream into the list of the blocking queues - if (result) { - amd::ScopedLock lock(streamSetLock); - streamSet.insert(this); - queue_ = queue; - queue->vdev()->profilerAttach(isProfilerAttached); - } else if (queue != nullptr) { - queue->release(); - } - - return result; -} - -// ================================================================================================ -amd::HostQueue* Stream::asHostQueue(bool skip_alloc) { - if (queue_ != nullptr) { - return queue_; - } - // Access to the stream object is lock protected, because possible allocation - amd::ScopedLock l(Lock()); - if (queue_ == nullptr) { - // Create the host queue for the first time - if (!skip_alloc) { - Create(); - } - } - return queue_; -} - -// ================================================================================================ -void Stream::Finish() const { - if (queue_ != nullptr) { - queue_->finish(); - } -} - -// ================================================================================================ -int Stream::DeviceId() const { - return device_->deviceId(); -} - -int Stream::DeviceId(const hipStream_t hStream) { - hip::Stream* s = reinterpret_cast(hStream); - int deviceId = (s != nullptr)? s->DeviceId() : ihipGetDevice(); - assert(deviceId >= 0 && deviceId < static_cast(g_devices.size())); - return deviceId; -} - -void Stream::syncNonBlockingStreams() { - amd::ScopedLock lock(streamSetLock); - for (auto& it : streamSet) { - if (it->Flags() & hipStreamNonBlocking) { - it->asHostQueue()->finish(); - } - } -} - -// ================================================================================================ -bool isValid(hipStream_t stream) { - // NULL stream is always valid - if (stream == nullptr) { - return true; - } - - hip::Stream* s = reinterpret_cast(stream); - amd::ScopedLock lock(streamSetLock); - if (streamSet.find(s) == streamSet.end()) { - return false; - } - return true; -} - -};// hip namespace - -// ================================================================================================ -void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null_stream) { - amd::Command::EventWaitList eventWaitList; - { - amd::ScopedLock lock(streamSetLock); - - for (const auto& stream : streamSet) { - amd::HostQueue* active_queue = stream->asHostQueue(); - // If it's the current device - if ((&active_queue->device() == &blocking_queue->device()) && - // Make sure it's a default stream - ((stream->Flags() & hipStreamNonBlocking) == 0) && - // and it's not the current stream - (active_queue != blocking_queue) && - // check for a wait on the null stream - (stream->Null() == wait_null_stream)) { - // Get the last valid command - amd::Command* command = active_queue->getLastQueuedCommand(true); - if (command != nullptr) { - // Check the current active status - if (command->status() != CL_COMPLETE) { - command->notifyCmdQueue(); - eventWaitList.push_back(command); - } else { - command->release(); - } - } - // Nullstream, hence there is nothing else to wait - if (wait_null_stream) { - break; - } - } - } - } - - // Check if we have to wait anything - if (eventWaitList.size() > 0) { - amd::Command* command = new amd::Marker(*blocking_queue, kMarkerDisableFlush, eventWaitList); - if (command != nullptr) { - command->enqueue(); - command->release(); - } - } - - // Release all active commands. It's safe after the marker was enqueued - for (const auto& it : eventWaitList) { - it->release(); - } -} - -// ================================================================================================ -void CL_CALLBACK ihipStreamCallback(cl_event event, cl_int command_exec_status, void* user_data) { - hipError_t status = hipSuccess; - StreamCallback* cbo = reinterpret_cast(user_data); - cbo->callBack_(cbo->stream_, status, cbo->userData_); - cbo->command_->release(); - delete cbo; -} - -// ================================================================================================ -static hipError_t ihipStreamCreate(hipStream_t* stream, - unsigned int flags, hip::Stream::Priority priority, - const std::vector& cuMask = {}) { - if (flags != hipStreamDefault && flags != hipStreamNonBlocking) { - return hipErrorInvalidValue; - } - hip::Stream* hStream = new hip::Stream(hip::getCurrentDevice(), priority, flags, false, cuMask); - - if (hStream == nullptr || !hStream->Create()) { - delete hStream; - return hipErrorOutOfMemory; - } - - *stream = reinterpret_cast(hStream); - - return hipSuccess; -} - -// ================================================================================================ -hipError_t hipStreamCreateWithFlags(hipStream_t *stream, unsigned int flags) { - HIP_INIT_API(hipStreamCreateWithFlags, stream, flags); - - if (stream == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(ihipStreamCreate(stream, flags, hip::Stream::Priority::Normal), *stream); -} - -// ================================================================================================ -hipError_t hipStreamCreate(hipStream_t *stream) { - HIP_INIT_API(hipStreamCreate, stream); - - if (stream == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(ihipStreamCreate(stream, hipStreamDefault, hip::Stream::Priority::Normal), *stream); -} - -// ================================================================================================ -hipError_t hipStreamCreateWithPriority(hipStream_t* stream, unsigned int flags, int priority) { - HIP_INIT_API(hipStreamCreateWithPriority, stream, flags, priority); - - if (stream == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - hip::Stream::Priority streamPriority; - if (priority <= hip::Stream::Priority::High) { - streamPriority = hip::Stream::Priority::High; - } else if (priority >= hip::Stream::Priority::Low) { - streamPriority = hip::Stream::Priority::Low; - } else { - streamPriority = hip::Stream::Priority::Normal; - } - - HIP_RETURN(ihipStreamCreate(stream, flags, streamPriority), *stream); -} - -// ================================================================================================ -hipError_t hipDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) { - HIP_INIT_API(hipDeviceGetStreamPriorityRange, leastPriority, greatestPriority); - - if (leastPriority != nullptr) { - *leastPriority = hip::Stream::Priority::Low; - } - if (greatestPriority != nullptr) { - *greatestPriority = hip::Stream::Priority::High; - } - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipStreamGetFlags(hipStream_t stream, unsigned int* flags) { - HIP_INIT_API(hipStreamGetFlags, stream, flags); - - if ((flags != nullptr) && (stream != nullptr)) { - if (!hip::isValid(stream)) { - return HIP_RETURN(hipErrorContextIsDestroyed); - } - *flags = reinterpret_cast(stream)->Flags(); - } else { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipStreamSynchronize(hipStream_t stream) { - HIP_INIT_API(hipStreamSynchronize, stream); - - if (!hip::isValid(stream)) { - return HIP_RETURN(hipErrorContextIsDestroyed); - } - - // Wait for the current host queue - hip::getQueue(stream)->finish(); - - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipStreamDestroy(hipStream_t stream) { - HIP_INIT_API(hipStreamDestroy, stream); - - if (stream == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - - if (!hip::isValid(stream)) { - return HIP_RETURN(hipErrorContextIsDestroyed); - } - - delete reinterpret_cast(stream); - - HIP_RETURN(hipSuccess); -} - -struct CallbackData { - int previous_read_index; - hip::ihipIpcEventShmem_t *shmem; -}; - -void WaitThenDecrementSignal(hipStream_t stream, hipError_t status, void* user_data){ - CallbackData *data = (CallbackData *)user_data; - int offset = data->previous_read_index % IPC_SIGNALS_PER_EVENT; - while (data->shmem->read_index < data->previous_read_index+IPC_SIGNALS_PER_EVENT - && data->shmem->signal[offset] != 0) { - } - delete data; -} - -// ================================================================================================ -hipError_t hipStreamWaitEvent(hipStream_t stream, hipEvent_t event, unsigned int flags) { - HIP_INIT_API(hipStreamWaitEvent, stream, event, flags); - - EVENT_CAPTURE(hipStreamWaitEvent, event, stream, flags); - - if (event == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - - if (!hip::isValid(stream)) { - return HIP_RETURN(hipErrorContextIsDestroyed); - } - - amd::HostQueue* queue = hip::getQueue(stream); - - hip::Event* e = reinterpret_cast(event); - if (e->flags & hipEventInterprocess) { - amd::Command* command = new amd::Marker(*queue, false); - auto t{new CallbackData{e->ipc_evt_.ipc_shmem_->read_index, e->ipc_evt_.ipc_shmem_}}; - StreamCallback* cbo = new StreamCallback(stream, - reinterpret_cast (WaitThenDecrementSignal), t, command); - if (!command->setCallback(CL_COMPLETE, ihipStreamCallback,cbo)) { - command->release(); - return hipErrorInvalidHandle; - } - command->enqueue(); - command->awaitCompletion(); - HIP_RETURN(hipSuccess); - } else { - HIP_RETURN(e->streamWait(queue, flags)); - } -} - -// ================================================================================================ -hipError_t hipStreamQuery(hipStream_t stream) { - HIP_INIT_API(hipStreamQuery, stream); - - if (!hip::isValid(stream)) { - return HIP_RETURN(hipErrorContextIsDestroyed); - } - - amd::HostQueue* hostQueue = hip::getQueue(stream); - - amd::Command* command = hostQueue->getLastQueuedCommand(true); - if (command == nullptr) { - // Nothing was submitted to the queue - HIP_RETURN(hipSuccess); - } - - amd::Event& event = command->event(); - if (command->type() != 0) { - event.notifyCmdQueue(); - } - hipError_t status = (command->status() == CL_COMPLETE) ? hipSuccess : hipErrorNotReady; - command->release(); - HIP_RETURN(status); -} - -// ================================================================================================ -hipError_t hipStreamAddCallback(hipStream_t stream, hipStreamCallback_t callback, void* userData, - unsigned int flags) { - HIP_INIT_API(hipStreamAddCallback, stream, callback, userData, flags); - //flags - Reserved for future use, must be 0 - if (callback == nullptr || flags != 0) { - HIP_RETURN(hipErrorInvalidValue); - } - - if (!hip::isValid(stream)) { - return HIP_RETURN(hipErrorContextIsDestroyed); - } - - amd::HostQueue* hostQueue = hip::getQueue(stream); - amd::Command* last_command = hostQueue->getLastQueuedCommand(true); - amd::Command::EventWaitList eventWaitList; - if (last_command != nullptr) { - eventWaitList.push_back(last_command); - } - amd::Command* command = new amd::Marker(*hostQueue, !kMarkerDisableFlush, eventWaitList); - if (command == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - StreamCallback* cbo = new StreamCallback(stream, callback, userData, command); - - if ((cbo == nullptr) || !command->setCallback(CL_COMPLETE, ihipStreamCallback, cbo)) { - command->release(); - if (last_command != nullptr) { - last_command->release(); - } - return hipErrorInvalidHandle; - } - // Retain callback command for the blocking marker - command->retain(); - command->enqueue(); - // @note: don't release the command here, because it will be released after HIP callback - if (last_command != nullptr) { - last_command->release(); - } - // Add the new barrier to stall the stream, until the callback is done - eventWaitList.clear(); - eventWaitList.push_back(command); - amd::Command* block_command = new amd::Marker(*hostQueue, !kMarkerDisableFlush, eventWaitList); - if (block_command == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - block_command->enqueue(); - block_command->release(); - // Release the callback marker - command->release(); - - HIP_RETURN(hipSuccess); -} - -// ================================================================================================ -hipError_t hipExtStreamCreateWithCUMask(hipStream_t* stream, uint32_t cuMaskSize, - const uint32_t* cuMask) { - HIP_INIT_API(hipExtStreamCreateWithCUMask, stream, cuMaskSize, cuMask); - - if (stream == nullptr) { - HIP_RETURN(hipErrorInvalidHandle); - } - if (cuMaskSize == 0 || cuMask == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - const std::vector cuMaskv(cuMask, cuMask + cuMaskSize); - - HIP_RETURN(ihipStreamCreate(stream, hipStreamDefault, hip::Stream::Priority::Normal, cuMaskv), *stream); -} - -// ================================================================================================ -hipError_t hipStreamGetPriority(hipStream_t stream, int* priority) { - HIP_INIT_API(hipStreamGetPriority, stream, priority); - if ((priority != nullptr) && (stream != nullptr)) { - if (!hip::isValid(stream)) { - return HIP_RETURN(hipErrorContextIsDestroyed); - } - *priority = static_cast(reinterpret_cast(stream)->GetPriority()); - } else { - HIP_RETURN(hipErrorInvalidValue); - } - - HIP_RETURN(hipSuccess); - -} - -// ================================================================================================ -hipError_t hipExtStreamGetCUMask(hipStream_t stream, uint32_t cuMaskSize, uint32_t* cuMask) { - HIP_INIT_API(hipExtStreamGetCUMask, stream, cuMaskSize, cuMask); - - if (cuMask == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - int deviceId = hip::getCurrentDevice()->deviceId(); - auto* deviceHandle = g_devices[deviceId]->devices()[0]; - const auto& info = deviceHandle->info(); - - // find the minimum cuMaskSize required to present the CU mask bit-array in a patch of 32 bits - // and return error if the cuMaskSize argument is less than cuMaskSizeRequired - uint32_t cuMaskSizeRequired = info.maxComputeUnits_ / 32 + - ((info.maxComputeUnits_ % 32) ? 1 : 0); - - if (cuMaskSize < cuMaskSizeRequired) { - HIP_RETURN(hipErrorInvalidValue); - } - - // make a default CU mask bit-array where all CUs are active - // this default mask will be returned when there is no - // custom or global CU mask defined - std::vector defaultCUMask; - uint32_t temp = 0; - uint32_t bit_index = 0; - for (uint32_t i = 0; i < info.maxComputeUnits_; i++) { - temp |= 1UL << bit_index; - if (bit_index >= 32) { - defaultCUMask.push_back(temp); - temp = 0; - bit_index = 0; - temp |= 1UL << bit_index; - } - bit_index += 1; - } - if (bit_index != 0) { - defaultCUMask.push_back(temp); - } - - // if the stream is null then either return globalCUMask_ (if it is defined) - // or return defaultCUMask - if (stream == nullptr) { - if (info.globalCUMask_.size() != 0) { - std::copy(info.globalCUMask_.begin(), info.globalCUMask_.end(), cuMask); - } else { - std::copy(defaultCUMask.begin(), defaultCUMask.end(), cuMask); - } - } else { - // if the stream is not null then get the stream's CU mask and return one of the below cases - // case1 if globalCUMask_ is defined then return the AND of globalCUMask_ and stream's CU mask - // case2 if globalCUMask_ is not defined then retuen AND of defaultCUMask and stream's CU mask - // in both cases above if stream's CU mask is empty then either globalCUMask_ (for case1) - // or defaultCUMask(for case2) will be returned - std::vector streamCUMask; - streamCUMask = reinterpret_cast(stream)->GetCUMask(); - std::vector mask = {}; - if (info.globalCUMask_.size() != 0) { - for (uint32_t i = 0; i < std::min(streamCUMask.size(), info.globalCUMask_.size()); i++) { - mask.push_back(streamCUMask[i] & info.globalCUMask_[i]); - } - } else { - for (uint32_t i = 0; i < std::min(streamCUMask.size(), defaultCUMask.size()); i++) { - mask.push_back(streamCUMask[i] & defaultCUMask[i]); - } - // check to make sure after ANDing streamCUMask (custom-defined) with global CU mask, - //we have non-zero mask, oterwise just return either globalCUMask_ or defaultCUMask - bool zeroCUMask = true; - for (auto m : mask) { - if (m != 0) { - zeroCUMask = false; - break; - } - } - if (zeroCUMask) { - mask = (info.globalCUMask_.size() != 0) ? info.globalCUMask_ : defaultCUMask; - } - std::copy(mask.begin(), mask.end(), cuMask); - } - } - HIP_RETURN(hipSuccess); -} diff --git a/src/hipamd/src/hip_stream_ops.cpp b/src/hipamd/src/hip_stream_ops.cpp deleted file mode 100644 index 06018e639c..0000000000 --- a/src/hipamd/src/hip_stream_ops.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include "hip_internal.hpp" -#include "platform/command_utils.hpp" - -hipError_t ihipStreamOperation(hipStream_t stream, cl_command_type cmdType, void* ptr, - int64_t value, uint64_t mask, unsigned int flags, size_t sizeBytes) { - size_t offset = 0; - unsigned int outFlags = 0; - - amd::Memory* memory = getMemoryObject(ptr, offset); - if (!memory) { - return hipErrorInvalidValue; - } - - // NOTE: 'mask' is only used in Wait operation, 'sizeBytes' is only used in Write operation - // 'flags' for now used only for Wait, but in future there will usecases for Write too. - - if (cmdType == ROCCLR_COMMAND_STREAM_WAIT_VALUE) { - // Wait is only supported on SignalMemory objects - if (!(memory->getMemFlags() & ROCCLR_MEM_HSA_SIGNAL_MEMORY)) { - return hipErrorInvalidValue; - } - switch (flags) { - case hipStreamWaitValueGte: - outFlags = ROCCLR_STREAM_WAIT_VALUE_GTE; - break; - case hipStreamWaitValueEq: - outFlags = ROCCLR_STREAM_WAIT_VALUE_EQ; - break; - case hipStreamWaitValueAnd: - outFlags = ROCCLR_STREAM_WAIT_VALUE_AND; - break; - case hipStreamWaitValueNor: - outFlags = ROCCLR_STREAM_WAIT_VALUE_NOR; - break; - default: - ShouldNotReachHere(); - break; - } - } else if (cmdType != ROCCLR_COMMAND_STREAM_WRITE_VALUE) { - return hipErrorInvalidValue; - } - - amd::HostQueue* queue = hip::getQueue(stream); - amd::Command::EventWaitList waitList; - - amd::StreamOperationCommand* command = - new amd::StreamOperationCommand(*queue, cmdType, waitList, *memory->asBuffer(), - value, mask, outFlags, offset, sizeBytes); - - if (command == nullptr) { - return hipErrorOutOfMemory; - } - command->enqueue(); - command->release(); - return hipSuccess; -} - -hipError_t hipStreamWaitValue32(hipStream_t stream, void* ptr, int32_t value, unsigned int flags, - uint32_t mask) { - HIP_INIT_API(hipStreamWaitValue32, stream, ptr, value, mask, flags); - // NOTE: ptr corresponds to a HSA Signal memeory which is 64 bits. - // 32 bit value and mask are converted to 64-bit values. - HIP_RETURN_DURATION(ihipStreamOperation( - stream, - ROCCLR_COMMAND_STREAM_WAIT_VALUE, - ptr, - value, - mask, - flags, - 0)); // sizeBytes un-used for wait, set it to 0 -} - -hipError_t hipStreamWaitValue64(hipStream_t stream, void* ptr, int64_t value, unsigned int flags, - uint64_t mask) { - HIP_INIT_API(hipStreamWaitValue64, stream, ptr, value, mask, flags); - HIP_RETURN_DURATION(ihipStreamOperation( - stream, - ROCCLR_COMMAND_STREAM_WAIT_VALUE, - ptr, - value, - mask, - flags, - 0)); // sizeBytes un-used for wait, set it to 0 -} - -hipError_t hipStreamWriteValue32(hipStream_t stream, void* ptr, int32_t value, unsigned int flags) { - HIP_INIT_API(hipStreamWriteValue32, stream, ptr, value, flags); - HIP_RETURN_DURATION(ihipStreamOperation( - stream, - ROCCLR_COMMAND_STREAM_WRITE_VALUE, - ptr, - value, - 0, // mask un-used set it to 0 - 0, // flags un-used for now set it to 0 - 4)); -} - -hipError_t hipStreamWriteValue64(hipStream_t stream, void* ptr, int64_t value, unsigned int flags) { - HIP_INIT_API(hipStreamWriteValue64, stream, ptr, value, flags); - HIP_RETURN_DURATION(ihipStreamOperation( - stream, - ROCCLR_COMMAND_STREAM_WRITE_VALUE, - ptr, - value, - 0, // mask un-used set it to 0 - 0, // flags un-used for now set it to 0 - 8)); -} diff --git a/src/hipamd/src/hip_surface.cpp b/src/hipamd/src/hip_surface.cpp deleted file mode 100644 index c2b2525fac..0000000000 --- a/src/hipamd/src/hip_surface.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include - -#include "hip_internal.hpp" -#include - -hipError_t hipCreateSurfaceObject(hipSurfaceObject_t* pSurfObject, - const hipResourceDesc* pResDesc) { - HIP_INIT_API(hipCreateSurfaceObject, pSurfObject, pResDesc); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipDestroySurfaceObject(hipSurfaceObject_t surfaceObject) { - HIP_INIT_API(hipDestroySurfaceObject, surfaceObject); - - HIP_RETURN(hipErrorNotSupported); -} diff --git a/src/hipamd/src/hip_texture.cpp b/src/hipamd/src/hip_texture.cpp deleted file mode 100755 index 2b97cea6d5..0000000000 --- a/src/hipamd/src/hip_texture.cpp +++ /dev/null @@ -1,1303 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include -#include "hip_internal.hpp" -#include "hip_platform.hpp" -#include "hip_conversions.hpp" -#include "platform/sampler.hpp" - -hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind, - amd::HostQueue& queue, bool isAsync = false); - -struct __hip_texture { - uint32_t imageSRD[HIP_IMAGE_OBJECT_SIZE_DWORD]; - uint32_t samplerSRD[HIP_SAMPLER_OBJECT_SIZE_DWORD]; - amd::Image* image; - amd::Sampler* sampler; - hipResourceDesc resDesc; - hipTextureDesc texDesc; - hipResourceViewDesc resViewDesc; - - __hip_texture(amd::Image* image_, - amd::Sampler* sampler_, - const hipResourceDesc& resDesc_, - const hipTextureDesc& texDesc_, - const hipResourceViewDesc& resViewDesc_) : - image(image_), - sampler(sampler_), - resDesc(resDesc_), - texDesc(texDesc_), - resViewDesc(resViewDesc_) { - amd::Context& context = *hip::getCurrentDevice()->asContext(); - amd::Device& device = *context.devices()[0]; - - device::Memory* imageMem = image->getDeviceMemory(device); - std::memcpy(imageSRD, imageMem->cpuSrd(), sizeof(imageSRD)); - - device::Sampler* samplerMem = sampler->getDeviceSampler(device); - std::memcpy(samplerSRD, samplerMem->hwState(), sizeof(samplerSRD)); - } -}; - -amd::Image* ihipImageCreate(const cl_channel_order channelOrder, - const cl_channel_type channelType, - const cl_mem_object_type imageType, - const size_t imageWidth, - const size_t imageHeight, - const size_t imageDepth, - const size_t imageArraySize, - const size_t imageRowPitch, - const size_t imageSlicePitch, - const uint32_t numMipLevels, - amd::Memory* buffer); - -hipError_t ihipCreateTextureObject(hipTextureObject_t* pTexObject, - const hipResourceDesc* pResDesc, - const hipTextureDesc* pTexDesc, - const hipResourceViewDesc* pResViewDesc) { - amd::Device* device = hip::getCurrentDevice()->devices()[0]; - const device::Info& info = device->info(); - - // Validate input params - if (pTexObject == nullptr || pResDesc == nullptr || pTexDesc == nullptr) { - return hipErrorInvalidValue; - } - - // pResViewDesc can only be specified if the type of resource is a HIP array or a HIP mipmapped array. - if ((pResViewDesc != nullptr) && - ((pResDesc->resType != hipResourceTypeArray) && (pResDesc->resType != hipResourceTypeMipmappedArray))) { - return hipErrorInvalidValue; - } - - // If hipResourceDesc::resType is set to hipResourceTypeArray, - // hipResourceDesc::res::array::array must be set to a valid HIP array handle. - if ((pResDesc->resType == hipResourceTypeArray) && - (pResDesc->res.array.array == nullptr)) { - return hipErrorInvalidValue; - } - - // If hipResourceDesc::resType is set to hipResourceTypeMipmappedArray, - // hipResourceDesc::res::mipmap::mipmap must be set to a valid HIP mipmapped array handle - // and hipTextureDesc::normalizedCoords must be set to true. - if ((pResDesc->resType == hipResourceTypeMipmappedArray) && - ((pResDesc->res.mipmap.mipmap == nullptr) || (pTexDesc->normalizedCoords == 0))) { - return hipErrorInvalidValue; - } - - // If hipResourceDesc::resType is set to hipResourceTypeLinear, - // hipResourceDesc::res::linear::devPtr must be set to a valid device pointer, that is aligned to hipDeviceProp::textureAlignment. - // The total number of elements in the linear address range cannot exceed hipDeviceProp::maxTexture1DLinear. - if ((pResDesc->resType == hipResourceTypeLinear) && - ((pResDesc->res.linear.devPtr == nullptr) || - (!amd::isMultipleOf(pResDesc->res.linear.devPtr, info.imageBaseAddressAlignment_)) || - ((pResDesc->res.linear.sizeInBytes / hip::getElementSize(pResDesc->res.linear.desc)) >= info.imageMaxBufferSize_))) { - return hipErrorInvalidValue; - } - - // If hipResourceDesc::resType is set to hipResourceTypePitch2D, - // hipResourceDesc::res::pitch2D::devPtr must be set to a valid device pointer, that is aligned to hipDeviceProp::textureAlignment. - // hipResourceDesc::res::pitch2D::width and hipResourceDesc::res::pitch2D::height specify the width and height of the array in elements, - // and cannot exceed hipDeviceProp::maxTexture2DLinear[0] and hipDeviceProp::maxTexture2DLinear[1] respectively. - // hipResourceDesc::res::pitch2D::pitchInBytes specifies the pitch between two rows in bytes and has to be aligned to hipDeviceProp::texturePitchAlignment. - // Pitch cannot exceed hipDeviceProp::maxTexture2DLinear[2]. - if ((pResDesc->resType == hipResourceTypePitch2D) && - ((pResDesc->res.pitch2D.devPtr == nullptr) || - (!amd::isMultipleOf(pResDesc->res.pitch2D.devPtr, info.imageBaseAddressAlignment_)) || - (pResDesc->res.pitch2D.width >= info.image2DMaxWidth_) || - (pResDesc->res.pitch2D.height >= info.image2DMaxHeight_) || - (!amd::isMultipleOf(pResDesc->res.pitch2D.pitchInBytes, info.imagePitchAlignment_)))) { - // TODO check pitch limits. - return hipErrorInvalidValue; - } - - // Mipmaps are currently not supported. - if (pResDesc->resType == hipResourceTypeMipmappedArray) { - return hipErrorNotSupported; - } - // We don't program the border_color_ptr field in the HW sampler SRD. - if (pTexDesc->addressMode[0] == hipAddressModeBorder) { - return hipErrorNotSupported; - } - // We don't program the max_ansio_ratio field in the the HW sampler SRD. - if (pTexDesc->maxAnisotropy != 0) { - return hipErrorNotSupported; - } - // We don't program the lod_bias field in the HW sampler SRD. - if (pTexDesc->mipmapLevelBias != 0) { - return hipErrorNotSupported; - } - // We don't program the min_lod field in the HW sampler SRD. - if (pTexDesc->minMipmapLevelClamp != 0) { - return hipErrorNotSupported; - } - // We don't program the max_lod field in the HW sampler SRD. - if (pTexDesc->maxMipmapLevelClamp != 0) { - return hipErrorNotSupported; - } - - // TODO ROCclr assumes all dimensions have the same addressing mode. - cl_addressing_mode addressMode = CL_ADDRESS_NONE; - // If hipTextureDesc::normalizedCoords is set to zero, - // hipAddressModeWrap and hipAddressModeMirror won't be supported - // and will be switched to hipAddressModeClamp. - if ((pTexDesc->normalizedCoords == 0) && - ((pTexDesc->addressMode[0] == hipAddressModeWrap) || (pTexDesc->addressMode[0] == hipAddressModeMirror))) { - addressMode = hip::getCLAddressingMode(hipAddressModeClamp); - } - // hipTextureDesc::addressMode is ignored if hipResourceDesc::resType is hipResourceTypeLinear - else if (pResDesc->resType != hipResourceTypeLinear) { - addressMode = hip::getCLAddressingMode(pTexDesc->addressMode[0]); - } - -#ifndef CL_FILTER_NONE -#define CL_FILTER_NONE 0x1142 -#endif - cl_filter_mode filterMode = CL_FILTER_NONE; -#undef CL_FILTER_NONE - // hipTextureDesc::filterMode is ignored if hipResourceDesc::resType is hipResourceTypeLinear. - if (pResDesc->resType != hipResourceTypeLinear) { - filterMode = hip::getCLFilterMode(pTexDesc->filterMode); - } - -#ifndef CL_FILTER_NONE -#define CL_FILTER_NONE 0x1142 -#endif - cl_filter_mode mipFilterMode = CL_FILTER_NONE; -#undef CL_FILTER_NONE - if (pResDesc->resType == hipResourceTypeMipmappedArray) { - mipFilterMode = hip::getCLFilterMode(pTexDesc->mipmapFilterMode); - } - - amd::Sampler* sampler = new amd::Sampler(*hip::getCurrentDevice()->asContext(), - pTexDesc->normalizedCoords, - addressMode, - filterMode, - mipFilterMode, - pTexDesc->minMipmapLevelClamp, - pTexDesc->maxMipmapLevelClamp); - - if (sampler == nullptr) { - return hipErrorOutOfMemory; - } - - if (!sampler->create()) { - delete sampler; - return hipErrorOutOfMemory; - } - - amd::Image* image = nullptr; - switch (pResDesc->resType) { - case hipResourceTypeArray: { - cl_mem memObj = reinterpret_cast(pResDesc->res.array.array->data); - if (!is_valid(memObj)) { - return hipErrorInvalidValue; - } - image = as_amd(memObj)->asImage(); - - hipTextureReadMode readMode = pTexDesc->readMode; - // 32-bit integer format will not be promoted, regardless of whether or not - // this hipTextureDesc::readMode is set hipReadModeNormalizedFloat is specified. - if ((pResDesc->res.array.array->Format == HIP_AD_FORMAT_SIGNED_INT32) || - (pResDesc->res.array.array->Format == HIP_AD_FORMAT_UNSIGNED_INT32)) { - readMode = hipReadModeElementType; - } - - // We need to create an image view if the user requested to use normalized pixel values, - // due to already having the image created with a different format. - if ((pResViewDesc != nullptr) || - (readMode == hipReadModeNormalizedFloat) || - (pTexDesc->sRGB == 1)) { - // TODO ROCclr currently right now can only change the format of the image. - const cl_channel_order channelOrder = (pResViewDesc != nullptr) ? hip::getCLChannelOrder(hip::getNumChannels(pResViewDesc->format), pTexDesc->sRGB) : - hip::getCLChannelOrder(pResDesc->res.array.array->NumChannels, pTexDesc->sRGB); - const cl_channel_type channelType = (pResViewDesc != nullptr) ? hip::getCLChannelType(hip::getArrayFormat(pResViewDesc->format), readMode) : - hip::getCLChannelType(pResDesc->res.array.array->Format, readMode); - const amd::Image::Format imageFormat(cl_image_format{channelOrder, channelType}); - if (!imageFormat.isValid()) { - return hipErrorInvalidValue; - } - - image = image->createView(*hip::getCurrentDevice()->asContext(), imageFormat, nullptr); - if (image == nullptr) { - return hipErrorInvalidValue; - } - } - break; - } - case hipResourceTypeMipmappedArray: { - ShouldNotReachHere(); - break; - } - case hipResourceTypeLinear: { - const cl_channel_order channelOrder = hip::getCLChannelOrder(hip::getNumChannels(pResDesc->res.linear.desc), pTexDesc->sRGB); - const cl_channel_type channelType = hip::getCLChannelType(hip::getArrayFormat(pResDesc->res.linear.desc), pTexDesc->readMode); - const amd::Image::Format imageFormat({channelOrder, channelType}); - const cl_mem_object_type imageType = hip::getCLMemObjectType(pResDesc->resType); - const size_t imageSizeInBytes = pResDesc->res.linear.sizeInBytes; - amd::Memory* buffer = getMemoryObjectWithOffset(pResDesc->res.linear.devPtr, imageSizeInBytes); - image = ihipImageCreate(channelOrder, - channelType, - imageType, - imageSizeInBytes / imageFormat.getElementSize(), /* imageWidth */ - 0, /* imageHeight */ - 0, /* imageDepth */ - 0, /* imageArraySize */ - 0, /* imageRowPitch */ - 0, /* imageSlicePitch */ - 0, /* numMipLevels */ - buffer); - buffer->release(); - if (image == nullptr) { - return hipErrorInvalidValue; - } - break; - } - case hipResourceTypePitch2D: { - const cl_channel_order channelOrder = hip::getCLChannelOrder(hip::getNumChannels(pResDesc->res.pitch2D.desc), pTexDesc->sRGB); - const cl_channel_type channelType = hip::getCLChannelType(hip::getArrayFormat(pResDesc->res.pitch2D.desc), pTexDesc->readMode); - const amd::Image::Format imageFormat({channelOrder, channelType}); - const cl_mem_object_type imageType = hip::getCLMemObjectType(pResDesc->resType); - const size_t imageSizeInBytes = pResDesc->res.pitch2D.width * imageFormat.getElementSize() + - pResDesc->res.pitch2D.pitchInBytes * (pResDesc->res.pitch2D.height - 1); - amd::Memory* buffer = getMemoryObjectWithOffset(pResDesc->res.pitch2D.devPtr, imageSizeInBytes); - image = ihipImageCreate(channelOrder, - channelType, - imageType, - pResDesc->res.pitch2D.width, /* imageWidth */ - pResDesc->res.pitch2D.height, /* imageHeight */ - 0, /* imageDepth */ - 0, /* imageArraySize */ - pResDesc->res.pitch2D.pitchInBytes, /* imageRowPitch */ - 0, /* imageSlicePitch */ - 0, /* numMipLevels */ - buffer); - buffer->release(); - if (image == nullptr) { - return hipErrorInvalidValue; - } - break; - } - } - - void *texObjectBuffer = nullptr; - ihipMalloc(&texObjectBuffer, sizeof(__hip_texture), CL_MEM_SVM_FINE_GRAIN_BUFFER); - if (texObjectBuffer == nullptr) { - return hipErrorOutOfMemory; - } - *pTexObject = new (texObjectBuffer) __hip_texture{image, sampler, *pResDesc, *pTexDesc, (pResViewDesc != nullptr) ? *pResViewDesc : hipResourceViewDesc{}}; - - return hipSuccess; -} - -hipError_t hipCreateTextureObject(hipTextureObject_t* pTexObject, - const hipResourceDesc* pResDesc, - const hipTextureDesc* pTexDesc, - const hipResourceViewDesc* pResViewDesc) { - HIP_INIT_API(hipCreateTextureObject, pTexObject, pResDesc, pTexDesc, pResViewDesc); - - HIP_RETURN(ihipCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc)); -} - - -hipError_t ihipDestroyTextureObject(hipTextureObject_t texObject) { - if (texObject == nullptr) { - return hipSuccess; - } - - const hipResourceType type = texObject->resDesc.resType; - const bool isImageFromBuffer = (type == hipResourceTypeLinear) || (type == hipResourceTypePitch2D); - const bool isImageView = ((type == hipResourceTypeArray) || (type == hipResourceTypeMipmappedArray)) && - !texObject->image->isParent(); - if (isImageFromBuffer || isImageView) { - texObject->image->release(); - } - - // TODO Should call ihipFree() to not polute the api trace. - return hipFree(texObject); -} - -hipError_t hipDestroyTextureObject(hipTextureObject_t texObject) { - HIP_INIT_API(hipDestroyTextureObject, texObject); - - HIP_RETURN(ihipDestroyTextureObject(texObject)); -} - - -hipError_t hipGetTextureObjectResourceDesc(hipResourceDesc* pResDesc, - hipTextureObject_t texObject) { - HIP_INIT_API(hipGetTextureObjectResourceDesc, pResDesc, texObject); - - if ((pResDesc == nullptr) || (texObject == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pResDesc = texObject->resDesc; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipGetTextureObjectResourceViewDesc(hipResourceViewDesc* pResViewDesc, - hipTextureObject_t texObject) { - HIP_INIT_API(hipGetTextureObjectResourceViewDesc, pResViewDesc, texObject); - - if ((pResViewDesc == nullptr) || (texObject == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pResViewDesc = texObject->resViewDesc; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipGetTextureObjectTextureDesc(hipTextureDesc* pTexDesc, - hipTextureObject_t texObject) { - HIP_INIT_API(hipGetTextureObjectTextureDesc, pTexDesc, texObject); - - if ((pTexDesc == nullptr) || (texObject == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pTexDesc = texObject->texDesc; - - HIP_RETURN(hipSuccess); -} - -inline bool ihipGetTextureAlignmentOffset(size_t* offset, - const void* devPtr) { - amd::Device* device = hip::getCurrentDevice()->devices()[0]; - const device::Info& info = device->info(); - - const char* alignedDevPtr = amd::alignUp(static_cast(devPtr), info.imageBaseAddressAlignment_); - const size_t alignedOffset = alignedDevPtr - static_cast(devPtr); - - // If the device memory pointer was returned from hipMalloc(), - // the offset is guaranteed to be 0 and NULL may be passed as the offset parameter. - if ((alignedOffset != 0) && (offset == nullptr)) { - LogPrintfError("Texture object not aligned with offset %u \n", alignedOffset); - return false; - } - - if (offset != nullptr) { - *offset = alignedOffset; - } - - return true; -} - -hipError_t ihipBindTexture(size_t* offset, - const textureReference* texref, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t size) { - if ((texref == nullptr) || - (devPtr == nullptr) || - (desc == nullptr)) { - return hipErrorInvalidValue; - } - - // Any previous address or HIP array state associated with the texture reference is superseded by this function. - // Any memory previously bound to hTexRef is unbound. - // No need to check for errors. - ihipDestroyTextureObject(texref->textureObject); - - hipResourceDesc resDesc = {}; - resDesc.resType = hipResourceTypeLinear; - resDesc.res.linear.devPtr = const_cast(devPtr); - resDesc.res.linear.desc = *desc; - resDesc.res.linear.sizeInBytes = size; - - if (ihipGetTextureAlignmentOffset(offset, devPtr)) { - // Align the user ptr to HW requirments. - resDesc.res.linear.devPtr = static_cast(const_cast(devPtr)) - *offset; - } else { - return hipErrorInvalidValue; - } - - hipTextureDesc texDesc = hip::getTextureDesc(texref); - - return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); -} - -hipError_t ihipBindTexture2D(size_t* offset, - const textureReference* texref, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t width, - size_t height, - size_t pitch) { - if ((texref == nullptr) || - (devPtr == nullptr) || - (desc == nullptr)) { - return hipErrorInvalidValue; - } - - // Any previous address or HIP array state associated with the texture reference is superseded by this function. - // Any memory previously bound to hTexRef is unbound. - // No need to check for errors. - ihipDestroyTextureObject(texref->textureObject); - - hipResourceDesc resDesc = {}; - resDesc.resType = hipResourceTypePitch2D; - resDesc.res.pitch2D.devPtr = const_cast(devPtr); - resDesc.res.pitch2D.desc = *desc; - resDesc.res.pitch2D.width = width; - resDesc.res.pitch2D.height = height; - resDesc.res.pitch2D.pitchInBytes = pitch; - - if (ihipGetTextureAlignmentOffset(offset, devPtr)) { - // Align the user ptr to HW requirments. - resDesc.res.pitch2D.devPtr = static_cast(const_cast(devPtr)) - *offset; - } else { - return hipErrorInvalidValue; - } - - hipTextureDesc texDesc = hip::getTextureDesc(texref); - - return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); -} - -hipError_t hipBindTexture2D(size_t* offset, - const textureReference* texref, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t width, - size_t height, - size_t pitch) { - HIP_INIT_API(hipBindTexture2D, offset, texref, devPtr, desc, width, height, pitch); - - hipDeviceptr_t refDevPtr = nullptr; - size_t refDevSize = 0; - - HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(texref, ihipGetDevice(), &refDevPtr, - &refDevSize)); - - assert(refDevSize == sizeof(textureReference)); - hipError_t err = ihipBindTexture2D(offset, texref, devPtr, desc, width, height, pitch); - if (err != hipSuccess) { - HIP_RETURN(err); - } - // Copy to device. - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN(ihipMemcpy(refDevPtr, texref, refDevSize, hipMemcpyHostToDevice, *queue)); -} - -hipError_t ihipBindTextureToArray(const textureReference* texref, - hipArray_const_t array, - const hipChannelFormatDesc* desc) { - if ((texref == nullptr) || - (array == nullptr) || - (desc == nullptr)) { - return hipErrorInvalidValue; - } - - // Any previous address or HIP array state associated with the texture reference is superseded by this function. - // Any memory previously bound to hTexRef is unbound. - // No need to check for errors. - ihipDestroyTextureObject(texref->textureObject); - - hipResourceDesc resDesc = {}; - resDesc.resType = hipResourceTypeArray; - resDesc.res.array.array = const_cast(array); - - hipTextureDesc texDesc = hip::getTextureDesc(texref); - - hipResourceViewFormat format = hip::getResourceViewFormat(*desc); - hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(array, format); - - return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, &resViewDesc); -} - -hipError_t hipBindTextureToArray(const textureReference* texref, - hipArray_const_t array, - const hipChannelFormatDesc* desc) { - HIP_INIT_API(hipBindTextureToArray, texref, array, desc); - - hipDeviceptr_t refDevPtr = nullptr; - size_t refDevSize = 0; - HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(texref, ihipGetDevice(), &refDevPtr, - &refDevSize)); - - assert(refDevSize == sizeof(textureReference)); - hipError_t err = ihipBindTextureToArray(texref, array, desc); - if (err != hipSuccess) { - HIP_RETURN(err); - } - // Copy to device. - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN(ihipMemcpy(refDevPtr, texref, refDevSize, hipMemcpyHostToDevice, *queue)); -} - -hipError_t ihipBindTextureToMipmappedArray(const textureReference* texref, - hipMipmappedArray_const_t mipmappedArray, - const hipChannelFormatDesc* desc) { - if ((texref == nullptr) || - (mipmappedArray == nullptr) || - (desc == nullptr)) { - return hipErrorInvalidValue; - } - - // Any previous address or HIP array state associated with the texture reference is superseded by this function. - // Any memory previously bound to hTexRef is unbound. - // No need to check for errors. - ihipDestroyTextureObject(texref->textureObject); - - hipResourceDesc resDesc = {}; - resDesc.resType = hipResourceTypeMipmappedArray; - resDesc.res.mipmap.mipmap = const_cast(mipmappedArray); - - hipTextureDesc texDesc = hip::getTextureDesc(texref); - - hipResourceViewFormat format = hip::getResourceViewFormat(*desc); - hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(mipmappedArray, format); - - return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, &resViewDesc); -} - -hipError_t hipBindTextureToMipmappedArray(const textureReference* texref, - hipMipmappedArray_const_t mipmappedArray, - const hipChannelFormatDesc* desc) { - HIP_INIT_API(hipBindTextureToMipmappedArray, texref, mipmappedArray, desc); - - hipDeviceptr_t refDevPtr = nullptr; - size_t refDevSize = 0; - - HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(texref, ihipGetDevice(), &refDevPtr, - &refDevSize)); - - assert(refDevSize == sizeof(textureReference)); - hipError_t err = ihipBindTextureToMipmappedArray(texref, mipmappedArray, desc); - if (err != hipSuccess) { - HIP_RETURN(err); - } - // Copy to device. - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN(ihipMemcpy(refDevPtr, texref, refDevSize, hipMemcpyHostToDevice, *queue)); -} - -hipError_t hipUnbindTexture(const textureReference* texref) { - HIP_INIT_API(hipUnbindTexture, texref); - - if (texref == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - const hipTextureObject_t textureObject = texref->textureObject; - const_cast(texref)->textureObject = nullptr; - - HIP_RETURN(ihipDestroyTextureObject(textureObject)); -} - -hipError_t hipBindTexture(size_t* offset, - const textureReference* texref, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t size) { - HIP_INIT_API(hipBindTexture, offset, texref, devPtr, desc, size); - - hipDeviceptr_t refDevPtr = nullptr; - size_t refDevSize = 0; - HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(texref, ihipGetDevice(), &refDevPtr, - &refDevSize)); - assert(refDevSize == sizeof(textureReference)); - hipError_t err = ihipBindTexture(offset, texref, devPtr, desc, size); - if (err != hipSuccess) { - HIP_RETURN(err); - } - // Copy to device. - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN(ihipMemcpy(refDevPtr, texref, refDevSize, hipMemcpyHostToDevice, *queue)); -} - -hipError_t hipGetChannelDesc(hipChannelFormatDesc* desc, - hipArray_const_t array) { - HIP_INIT_API(hipGetChannelDesc, desc, array); - - if ((desc == nullptr) || (array == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - // It is UB to call hipGetChannelDesc() on an array created via hipArrayCreate()/hipArray3DCreate(). - // This is due to hip not differentiating between runtime and driver types. - *desc = array->desc; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipGetTextureAlignmentOffset(size_t* offset, - const textureReference* texref) { - HIP_INIT_API(hipGetTextureAlignmentOffset, offset, texref); - - if ((offset == nullptr) || (texref == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - // TODO enforce alignment on devPtr. - *offset = 0; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipGetTextureReference(const textureReference** texref, const void* symbol) { - HIP_INIT_API(hipGetTextureReference, texref, symbol); - - if (texref == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - *texref = reinterpret_cast(symbol); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetFormat(textureReference* texRef, - hipArray_Format fmt, - int NumPackedComponents) { - HIP_INIT_API(hipTexRefSetFormat, texRef, fmt, NumPackedComponents); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - texRef->format = fmt; - texRef->numChannels = NumPackedComponents; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetFlags(textureReference* texRef, - unsigned int Flags) { - HIP_INIT_API(hipTexRefSetFlags, texRef, Flags); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - texRef->readMode = hipReadModeNormalizedFloat; - texRef->normalized = 0; - texRef->sRGB = 0; - - if (Flags & HIP_TRSF_READ_AS_INTEGER) { - texRef->readMode = hipReadModeElementType; - } - - if (Flags & HIP_TRSF_NORMALIZED_COORDINATES) { - texRef->normalized = 1; - } - - if (Flags & HIP_TRSF_SRGB) { - texRef->sRGB = 1; - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetFilterMode(textureReference* texRef, - hipTextureFilterMode fm) { - HIP_INIT_API(hipTexRefSetFilterMode, texRef, fm); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - texRef->filterMode = fm; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefGetAddressMode(hipTextureAddressMode* pam, - const textureReference* texRef, - int dim) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetAddressMode, pam, texRef, dim); - - if ((pam == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - // Currently, the only valid value for dim are 0 and 1. - if ((dim != 0) && (dim != 1)) { - LogPrintfError( - "Currently only 2 dimensions (0,1) are valid," - "dim : %d \n", - dim); - HIP_RETURN(hipErrorInvalidValue); - } - - *pam = texRef->addressMode[dim]; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetAddressMode(textureReference* texRef, - int dim, - hipTextureAddressMode am) { - HIP_INIT_API(hipTexRefSetAddressMode, texRef, dim, am); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - if ((dim < 0) || (dim > 2)) { - LogPrintfError( - "Currently only 3 dimensions (0,1,2) are valid," - "dim : %d \n", - dim); - HIP_RETURN(hipErrorInvalidValue); - } - - texRef->addressMode[dim] = am; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefGetArray(hipArray_t* pArray, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetArray, pArray, texRef); - - if ((pArray == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipResourceDesc resDesc = {}; - // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. - hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef->textureObject); - if (error != hipSuccess) { - HIP_RETURN(error); - } - - switch (resDesc.resType) { - case hipResourceTypeLinear: - case hipResourceTypePitch2D: - case hipResourceTypeMipmappedArray: { - HIP_RETURN(hipErrorInvalidValue); - } - case hipResourceTypeArray: - *pArray = resDesc.res.array.array; - break; - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetArray(textureReference* texRef, - hipArray_const_t array, - unsigned int flags) { - HIP_INIT_API(hipTexRefSetArray, texRef, array, flags); - - if ((texRef == nullptr) || (array == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - if (flags != HIP_TRSA_OVERRIDE_FORMAT) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipDeviceptr_t refDevPtr = nullptr; - size_t refDevSize = 0; - - HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, &refDevPtr, &refDevSize)); - assert(refDevSize == sizeof(textureReference)); - - // Any previous address or HIP array state associated with the texture reference is superseded by this function. - // Any memory previously bound to hTexRef is unbound. - // No need to check for errors. - ihipDestroyTextureObject(texRef->textureObject); - - hipResourceDesc resDesc = {}; - resDesc.resType = hipResourceTypeArray; - resDesc.res.array.array = const_cast(array); - - hipTextureDesc texDesc = hip::getTextureDesc(texRef); - - hipResourceViewFormat format = hip::getResourceViewFormat(hip::getChannelFormatDesc(texRef->numChannels, texRef->format)); - hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(array, format); - - hipError_t err = ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, &resViewDesc); - if (err != hipSuccess) { - HIP_RETURN(err); - } - // Copy to device. - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN(ihipMemcpy(refDevPtr, texRef, refDevSize, hipMemcpyHostToDevice, *queue)); -} - -hipError_t hipTexRefGetAddress(hipDeviceptr_t* dptr, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetAddress, dptr, texRef); - - if ((dptr == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipResourceDesc resDesc = {}; - // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. - hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef->textureObject); - if (error != hipSuccess) { - LogPrintfError("hipGetTextureObjectResourceDesc failed with error code: %s \n", - hipGetErrorName(error)); - HIP_RETURN(error); - } - - switch (resDesc.resType) { - // Need to verify. - // If the texture reference is not bound to any device memory range, - // return hipErroInvalidValue. - case hipResourceTypeArray: - case hipResourceTypeMipmappedArray: { - HIP_RETURN(hipErrorInvalidValue); - } - case hipResourceTypeLinear: - *dptr = resDesc.res.linear.devPtr; - break; - case hipResourceTypePitch2D: - *dptr = resDesc.res.pitch2D.devPtr; - break; - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetAddress(size_t* ByteOffset, - textureReference* texRef, - hipDeviceptr_t dptr, - size_t bytes) { - HIP_INIT_API(hipTexRefSetAddress, ByteOffset, texRef, dptr, bytes); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipDeviceptr_t refDevPtr = nullptr; - size_t refDevSize = 0; - HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, &refDevPtr, &refDevSize)); - assert(refDevSize == sizeof(textureReference)); - - // Any previous address or HIP array state associated with the texture reference is superseded by this function. - // Any memory previously bound to hTexRef is unbound. - // No need to check for errors. - ihipDestroyTextureObject(texRef->textureObject); - - hipResourceDesc resDesc = {}; - resDesc.resType = hipResourceTypeLinear; - resDesc.res.linear.devPtr = dptr; - resDesc.res.linear.desc = hip::getChannelFormatDesc(texRef->numChannels, texRef->format); - resDesc.res.linear.sizeInBytes = bytes; - - if (ihipGetTextureAlignmentOffset(ByteOffset, dptr)) { - // Align the user ptr to HW requirments. - resDesc.res.linear.devPtr = static_cast(dptr) - *ByteOffset; - } else { - HIP_RETURN(hipErrorInvalidValue); - } - - hipTextureDesc texDesc = hip::getTextureDesc(texRef); - - hipError_t err = ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, nullptr); - if (err != hipSuccess) { - HIP_RETURN(err); - } - // Copy to device. - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN(ihipMemcpy(refDevPtr, texRef, refDevSize, hipMemcpyHostToDevice, *queue)); -} - -hipError_t hipTexRefSetAddress2D(textureReference* texRef, - const HIP_ARRAY_DESCRIPTOR* desc, - hipDeviceptr_t dptr, - size_t Pitch) { - HIP_INIT_API(hipTexRefSetAddress2D, texRef, desc, dptr, Pitch); - - if ((texRef == nullptr) || (desc == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipDeviceptr_t refDevPtr = nullptr; - size_t refDevSize = 0; - HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, &refDevPtr, &refDevSize)); - assert(refDevSize == sizeof(textureReference)); - - // Any previous address or HIP array state associated with the texture reference is superseded by this function. - // Any memory previously bound to hTexRef is unbound. - // No need to check for errors. - ihipDestroyTextureObject(texRef->textureObject); - - hipResourceDesc resDesc = {}; - resDesc.resType = hipResourceTypePitch2D; - resDesc.res.linear.devPtr = dptr; - resDesc.res.linear.desc = hip::getChannelFormatDesc(desc->NumChannels, desc->Format); // Need to verify. - resDesc.res.pitch2D.width = desc->Width; - resDesc.res.pitch2D.height = desc->Height; - resDesc.res.pitch2D.pitchInBytes = Pitch; - - hipTextureDesc texDesc = hip::getTextureDesc(texRef); - - hipError_t err = ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, nullptr); - if (err != hipSuccess) { - HIP_RETURN(err); - } - // Copy to device. - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN(ihipMemcpy(refDevPtr, texRef, refDevSize, hipMemcpyHostToDevice, *queue)); -} - -hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f) { - return {x, y, z, w, f}; -} - -hipError_t hipTexRefGetBorderColor(float* pBorderColor, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetBorderColor, pBorderColor, texRef); - - if ((pBorderColor == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - // TODO add textureReference::borderColor. - assert(false && "textureReference::borderColor is missing in header"); - // std::memcpy(pBorderColor, texRef.borderColor, sizeof(texRef.borderColor)); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefGetFilterMode(hipTextureFilterMode* pfm, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetFilterMode, pfm, texRef); - - if ((pfm == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pfm = texRef->filterMode; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefGetFlags(unsigned int* pFlags, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetFlags, pFlags, texRef); - - if ((pFlags == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pFlags = 0; - - if (texRef->readMode == hipReadModeElementType) { - *pFlags |= HIP_TRSF_READ_AS_INTEGER; - } - - if (texRef->normalized == 1) { - *pFlags |= HIP_TRSF_NORMALIZED_COORDINATES; - } - - if (texRef->sRGB == 1) { - *pFlags |= HIP_TRSF_SRGB; - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefGetFormat(hipArray_Format* pFormat, - int* pNumChannels, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetFormat, pFormat, pNumChannels, texRef); - - if ((pFormat == nullptr) || (pNumChannels == nullptr) || - (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pFormat = texRef->format; - *pNumChannels = texRef->numChannels; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefGetMaxAnisotropy(int* pmaxAnsio, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMaxAnisotropy, pmaxAnsio, texRef); - - if ((pmaxAnsio == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pmaxAnsio = texRef->maxAnisotropy; - - HIP_RETURN(hipErrorInvalidValue); -} - -hipError_t hipTexRefGetMipmapFilterMode(hipTextureFilterMode* pfm, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMipmapFilterMode, pfm, texRef); - - if ((pfm == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pfm = texRef->mipmapFilterMode; - - HIP_RETURN(hipErrorInvalidValue); -} - -hipError_t hipTexRefGetMipmapLevelBias(float* pbias, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMipmapLevelBias, pbias, texRef); - - if ((pbias == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pbias = texRef->mipmapLevelBias; - - HIP_RETURN(hipErrorInvalidValue); -} - -hipError_t hipTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, - float* pmaxMipmapLevelClamp, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMipmapLevelClamp, pminMipmapLevelClamp, pmaxMipmapLevelClamp, texRef); - - if ((pminMipmapLevelClamp == nullptr) || (pmaxMipmapLevelClamp == nullptr) || - (texRef == nullptr)){ - HIP_RETURN(hipErrorInvalidValue); - } - - *pminMipmapLevelClamp = texRef->minMipmapLevelClamp; - *pmaxMipmapLevelClamp = texRef->maxMipmapLevelClamp; - - HIP_RETURN(hipErrorInvalidValue); -} - -hipError_t hipTexRefGetMipmappedArray(hipMipmappedArray_t* pArray, - const textureReference* texRef) { - // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMipmappedArray, pArray, &texRef); - - if ((pArray == nullptr) || (texRef == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipResourceDesc resDesc = {}; - // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. - hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef->textureObject); - if (error != hipSuccess) { - HIP_RETURN(error); - } - - switch (resDesc.resType) { - case hipResourceTypeLinear: - case hipResourceTypePitch2D: - case hipResourceTypeArray: { - HIP_RETURN(hipErrorInvalidValue); - } - case hipResourceTypeMipmappedArray: - *pArray = resDesc.res.mipmap.mipmap; - break; - } - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetBorderColor(textureReference* texRef, - float* pBorderColor) { - HIP_INIT_API(hipTexRefSetBorderColor, texRef, pBorderColor); - - if ((texRef == nullptr) || (pBorderColor == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - // TODO add textureReference::borderColor. - assert(false && "textureReference::borderColor is missing in header"); - // std::memcpy(texRef.borderColor, pBorderColor, sizeof(texRef.borderColor)); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetMaxAnisotropy(textureReference* texRef, - unsigned int maxAniso) { - HIP_INIT_API(hipTexRefSetMaxAnisotropy, texRef, maxAniso); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - texRef->maxAnisotropy = maxAniso; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetMipmapFilterMode(textureReference* texRef, - hipTextureFilterMode fm) { - HIP_INIT_API(hipTexRefSetMipmapFilterMode, texRef, fm); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - texRef->mipmapFilterMode = fm; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetMipmapLevelBias(textureReference* texRef, - float bias) { - HIP_INIT_API(hipTexRefSetMipmapLevelBias, texRef, bias); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - texRef->mipmapLevelBias = bias; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetMipmapLevelClamp(textureReference* texRef, - float minMipMapLevelClamp, - float maxMipMapLevelClamp) { - HIP_INIT_API(hipTexRefSetMipmapLevelClamp, minMipMapLevelClamp, maxMipMapLevelClamp); - - if (texRef == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - texRef->minMipmapLevelClamp = minMipMapLevelClamp; - texRef->maxMipmapLevelClamp = maxMipMapLevelClamp; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetMipmappedArray(textureReference* texRef, - hipMipmappedArray* mipmappedArray, - unsigned int Flags) { - HIP_INIT_API(hipTexRefSetMipmappedArray, texRef, mipmappedArray, Flags); - - if ((texRef == nullptr) || (mipmappedArray == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - if (Flags != HIP_TRSA_OVERRIDE_FORMAT) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipDeviceptr_t refDevPtr = nullptr; - size_t refDevSize = 0; - HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, &refDevPtr, &refDevSize)); - assert(refDevSize == sizeof(textureReference)); - - // Any previous address or HIP array state associated with the texture reference is superseded by this function. - // Any memory previously bound to hTexRef is unbound. - // No need to check for errors. - ihipDestroyTextureObject(texRef->textureObject); - - hipResourceDesc resDesc = {}; - resDesc.resType = hipResourceTypeMipmappedArray; - resDesc.res.mipmap.mipmap = mipmappedArray; - - hipTextureDesc texDesc = hip::getTextureDesc(texRef); - - hipResourceViewFormat format = hip::getResourceViewFormat(hip::getChannelFormatDesc(texRef->numChannels, texRef->format)); - hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(mipmappedArray, format); - - hipError_t err = ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, &resViewDesc); - if (err != hipSuccess) { - HIP_RETURN(err); - } - // Copy to device. - amd::HostQueue* queue = hip::getNullStream(); - HIP_RETURN(ihipMemcpy(refDevPtr, texRef, refDevSize, hipMemcpyHostToDevice, *queue)); -} - -hipError_t hipTexObjectCreate(hipTextureObject_t* pTexObject, - const HIP_RESOURCE_DESC* pResDesc, - const HIP_TEXTURE_DESC* pTexDesc, - const HIP_RESOURCE_VIEW_DESC* pResViewDesc) { - HIP_INIT_API(hipTexObjectCreate, pTexObject, pResDesc, pTexDesc, pResViewDesc); - - if ((pTexObject == nullptr) || (pResDesc == nullptr) || (pTexDesc == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - hipResourceDesc resDesc = hip::getResourceDesc(*pResDesc); - hipTextureDesc texDesc = hip::getTextureDesc(*pTexDesc); - - if (pResViewDesc != nullptr) { - hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(*pResViewDesc); - HIP_RETURN(ihipCreateTextureObject(pTexObject, &resDesc, &texDesc, &resViewDesc)); - } else { - HIP_RETURN(ihipCreateTextureObject(pTexObject, &resDesc, &texDesc, nullptr)); - } -} - -hipError_t hipTexObjectDestroy(hipTextureObject_t texObject) { - HIP_INIT_API(hipTexObjectDestroy, texObject); - - HIP_RETURN(ihipDestroyTextureObject(texObject)); -} - -hipError_t hipTexObjectGetResourceDesc(HIP_RESOURCE_DESC* pResDesc, - hipTextureObject_t texObject) { - HIP_INIT_API(hipTexObjectGetResourceDesc, pResDesc, texObject); - - if ((pResDesc == nullptr) || (texObject == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pResDesc = hip::getResourceDesc(texObject->resDesc); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexObjectGetResourceViewDesc(HIP_RESOURCE_VIEW_DESC* pResViewDesc, - hipTextureObject_t texObject) { - HIP_INIT_API(hipTexObjectGetResourceViewDesc, pResViewDesc, texObject); - - if ((pResViewDesc == nullptr) || (texObject == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pResViewDesc = hip::getResourceViewDesc(texObject->resViewDesc); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexObjectGetTextureDesc(HIP_TEXTURE_DESC* pTexDesc, - hipTextureObject_t texObject) { - HIP_INIT_API(hipTexObjectGetTextureDesc, pTexDesc, texObject); - - if ((pTexDesc == nullptr) || (texObject == nullptr)) { - HIP_RETURN(hipErrorInvalidValue); - } - - *pTexDesc = hip::getTextureDesc(texObject->texDesc); - - HIP_RETURN(hipSuccess); -} diff --git a/src/hipamd/src/hiprtc_internal.hpp b/src/hipamd/src/hiprtc_internal.hpp deleted file mode 100644 index 4e533c9716..0000000000 --- a/src/hipamd/src/hiprtc_internal.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef HIPRTC_SRC_HIP_INTERNAL_H -#define HIPRTC_SRC_HIP_INTERNAL_H - -#include "hip_internal.hpp" - -#if __linux__ -#include - -#if HIPRTC_USE_CXXABI -#include - -#define DEMANGLE abi::__cxa_demangle - -#else -extern "C" char * __cxa_demangle(const char *mangled_name, char *output_buffer, - size_t *length, int *status); - -#define DEMANGLE __cxa_demangle -#endif //HIPRTC_USE_CXXABI - -#elif defined(_WIN32) -#include -#include - -#define UNDECORATED_SIZE 4096 - -#endif // __linux__ - -// This macro should be called at the beginning of every HIP RTC API. -#define HIPRTC_INIT_API(...) \ - ClPrint(amd::LOG_INFO, amd::LOG_API, "[%zx] %s ( %s )", std::this_thread::get_id(), __func__, ToString( __VA_ARGS__ ).c_str()); \ - amd::Thread* thread = amd::Thread::current(); \ - if (!VDI_CHECK_THREAD(thread)) { \ - HIPRTC_RETURN(HIPRTC_ERROR_INTERNAL_ERROR); \ - } \ - HIP_INIT(); - -#define HIPRTC_RETURN(ret) \ - hiprtc::g_lastRtcError = ret; \ - ClPrint(amd::LOG_INFO, amd::LOG_API, "[%zx] %s: Returned %s", std::this_thread::get_id(), __func__, \ - hiprtcGetErrorString(hiprtc::g_lastRtcError)); \ - return hiprtc::g_lastRtcError; - - -#endif // HIPRTC_SRC_HIP_INTERNAL_H diff --git a/src/hipamd/src/trace_helper.h b/src/hipamd/src/trace_helper.h deleted file mode 100644 index ee4eae68ac..0000000000 --- a/src/hipamd/src/trace_helper.h +++ /dev/null @@ -1,246 +0,0 @@ -/* Copyright (c) 2015-present Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#pragma once - -#include -#include -#include -#include -//--- -// Helper functions to convert HIP function arguments into strings. -// Handles POD data types as well as enumerations (ie hipMemcpyKind). -// The implementation uses C++11 variadic templates and template specialization. -// The hipMemcpyKind example below is a good example that shows how to implement conversion for a -// new HSA type. - - -// Handy macro to convert an enumeration to a stringified version of same: -#define CASE_STR(x) \ - case x: \ - return #x; - -inline const char* ihipErrorString(hipError_t hip_error) { - switch (hip_error) { - CASE_STR(hipSuccess); - CASE_STR(hipErrorOutOfMemory); - CASE_STR(hipErrorNotInitialized); - CASE_STR(hipErrorDeinitialized); - CASE_STR(hipErrorProfilerDisabled); - CASE_STR(hipErrorProfilerNotInitialized); - CASE_STR(hipErrorProfilerAlreadyStarted); - CASE_STR(hipErrorProfilerAlreadyStopped); - CASE_STR(hipErrorInvalidImage); - CASE_STR(hipErrorInvalidContext); - CASE_STR(hipErrorContextAlreadyCurrent); - CASE_STR(hipErrorMapFailed); - CASE_STR(hipErrorUnmapFailed); - CASE_STR(hipErrorArrayIsMapped); - CASE_STR(hipErrorAlreadyMapped); - CASE_STR(hipErrorNoBinaryForGpu); - CASE_STR(hipErrorAlreadyAcquired); - CASE_STR(hipErrorNotMapped); - CASE_STR(hipErrorNotMappedAsArray); - CASE_STR(hipErrorNotMappedAsPointer); - CASE_STR(hipErrorECCNotCorrectable); - CASE_STR(hipErrorUnsupportedLimit); - CASE_STR(hipErrorContextAlreadyInUse); - CASE_STR(hipErrorPeerAccessUnsupported); - CASE_STR(hipErrorInvalidKernelFile); - CASE_STR(hipErrorInvalidGraphicsContext); - CASE_STR(hipErrorInvalidSource); - CASE_STR(hipErrorFileNotFound); - CASE_STR(hipErrorSharedObjectSymbolNotFound); - CASE_STR(hipErrorSharedObjectInitFailed); - CASE_STR(hipErrorOperatingSystem); - CASE_STR(hipErrorSetOnActiveProcess); - CASE_STR(hipErrorInvalidHandle); - CASE_STR(hipErrorNotFound); - CASE_STR(hipErrorIllegalAddress); - CASE_STR(hipErrorMissingConfiguration); - CASE_STR(hipErrorLaunchFailure); - CASE_STR(hipErrorPriorLaunchFailure); - CASE_STR(hipErrorLaunchTimeOut); - CASE_STR(hipErrorLaunchOutOfResources); - CASE_STR(hipErrorInvalidDeviceFunction); - CASE_STR(hipErrorInvalidConfiguration); - CASE_STR(hipErrorInvalidDevice); - CASE_STR(hipErrorInvalidValue); - CASE_STR(hipErrorInvalidPitchValue); - CASE_STR(hipErrorInvalidDevicePointer); - CASE_STR(hipErrorInvalidMemcpyDirection); - CASE_STR(hipErrorUnknown); - CASE_STR(hipErrorNotReady); - CASE_STR(hipErrorNoDevice); - CASE_STR(hipErrorPeerAccessAlreadyEnabled); - CASE_STR(hipErrorPeerAccessNotEnabled); - CASE_STR(hipErrorRuntimeMemory); - CASE_STR(hipErrorRuntimeOther); - CASE_STR(hipErrorHostMemoryAlreadyRegistered); - CASE_STR(hipErrorHostMemoryNotRegistered); - CASE_STR(hipErrorTbd); - default: - return "hipErrorUnknown"; - }; -}; - -// Building block functions: -template -inline std::string ToHexString(T v) { - std::ostringstream ss; - ss << "0x" << std::hex << v; - return ss.str(); -}; - -template -inline std::string ToString(T* v) { - std::ostringstream ss; - if (v == NULL) { - ss << "char array:"; - } else { - ss << v; - } - return ss.str(); -}; - -template -inline std::string ToString(T** v) { - std::ostringstream ss; - if (v == NULL) { - ss << "char array:"; - } else { - ss << v; - } - return ss.str(); -}; - -//--- -// Template overloads for ToString to handle specific types - -// This is the default which works for most types: -template -inline std::string ToString(T v) { - std::ostringstream ss; - ss << v; - return ss.str(); -}; - -template <> -inline std::string ToString(hipFunction_t v) { - std::ostringstream ss; - ss << "0x" << std::hex << static_cast(v); - return ss.str(); -}; - -// hipEvent_t specialization. TODO - maybe add an event ID for debug? -template <> -inline std::string ToString(hipEvent_t v) { - std::ostringstream ss; - ss << "event:" << std::hex << static_cast(v); - return ss.str(); -}; -// hipStream_t -template <> -inline std::string ToString(hipStream_t v) { - std::ostringstream ss; - if (v == NULL) { - ss << "stream:"; - } else { - ss << "stream:" << std::hex << static_cast(v); - } - - return ss.str(); -}; - -// hipCtx_t -template <> -inline std::string ToString(hipCtx_t v) { - std::ostringstream ss; - if (v == NULL) { - ss << "context:"; - } else { - ss << "context:" << std::hex << static_cast(v); - } - - return ss.str(); -}; - -// hipPitchedPtr -template <> -inline std::string ToString(hipPitchedPtr v) { - std::ostringstream ss; - ss << "pitchPtr:" << std::hex << static_cast(v.ptr); - return ss.str(); -}; - -// hipMemcpyKind specialization -template <> -inline std::string ToString(hipMemcpyKind v) { - switch (v) { - CASE_STR(hipMemcpyHostToHost); - CASE_STR(hipMemcpyHostToDevice); - CASE_STR(hipMemcpyDeviceToHost); - CASE_STR(hipMemcpyDeviceToDevice); - CASE_STR(hipMemcpyDefault); - default: - return ToHexString(v); - }; -}; - -template <> -inline std::string ToString(hipFuncCache_t v) { - switch (v) { - CASE_STR(hipFuncCachePreferNone); - CASE_STR(hipFuncCachePreferShared); - CASE_STR(hipFuncCachePreferL1); - CASE_STR(hipFuncCachePreferEqual); - default: - return ToHexString(v); - }; -}; - -template <> -inline std::string ToString(hipSharedMemConfig v) { - switch (v) { - CASE_STR(hipSharedMemBankSizeDefault); - CASE_STR(hipSharedMemBankSizeFourByte); - CASE_STR(hipSharedMemBankSizeEightByte); - default: - return ToHexString(v); - }; -}; - -template <> -inline std::string ToString(hipError_t v) { - return ihipErrorString(v); -}; - -// Catch empty arguments case -inline std::string ToString() { return (""); } - - -//--- -// C++11 variadic template - peels off first argument, converts to string, and calls itself again to -// peel the next arg. Strings are automatically separated by comma+space. -template -inline std::string ToString(T first, Args... args) { - return ToString(first) + ", " + ToString(args...); -} -