From 6e867eacb6a8d970993b27c99853e47b3841da68 Mon Sep 17 00:00:00 2001 From: Aryan Salmanpour Date: Thu, 30 Jan 2020 15:05:53 -0500 Subject: [PATCH 001/154] [HIP][HIPIFY] Add some missing flags for cooperative launch and occupancy APIs --- bin/hipify-perl | 5 +++++ .../CUDA_Driver_API_functions_supported_by_HIP.md | 5 +++-- .../CUDA_Runtime_API_functions_supported_by_HIP.md | 4 ++-- hipify-clang/src/CUDA2HIP_Driver_API_types.cpp | 2 ++ hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp | 2 +- include/hip/hcc_detail/hip_runtime_api.h | 11 ++++++++--- include/hip/hip_runtime_api.h | 4 ++++ include/hip/nvcc_detail/hip_runtime_api.h | 12 ++++++++++++ src/hip_hcc.cpp | 2 ++ 9 files changed, 39 insertions(+), 8 deletions(-) diff --git a/bin/hipify-perl b/bin/hipify-perl index 4fe9b80349..d18e227c3d 100755 --- a/bin/hipify-perl +++ b/bin/hipify-perl @@ -1186,6 +1186,7 @@ sub simpleSubstitutions { $ft{'numeric_literal'} += s/\bCUDA_ERROR_INVALID_PTX\b/hipErrorInvalidKernelFile/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_INVALID_SOURCE\b/hipErrorInvalidSource/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_INVALID_VALUE\b/hipErrorInvalidValue/g; + $ft{'numeric_literal'} += s/\bCUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE\b/hipErrorCooperativeLaunchTooLarge/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_LAUNCH_FAILED\b/hipErrorLaunchFailure/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_LAUNCH_OUT_OF_RESOURCES\b/hipErrorLaunchOutOfResources/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_LAUNCH_TIMEOUT\b/hipErrorLaunchTimeOut/g; @@ -1659,6 +1660,7 @@ sub simpleSubstitutions { $ft{'numeric_literal'} += s/\bcudaErrorInvalidSource\b/hipErrorInvalidSource/g; $ft{'numeric_literal'} += s/\bcudaErrorInvalidSymbol\b/hipErrorInvalidSymbol/g; $ft{'numeric_literal'} += s/\bcudaErrorInvalidValue\b/hipErrorInvalidValue/g; + $ft{'numeric_literal'} += s/\bcudaErrorCooperativeLaunchTooLarge\b/hipErrorCooperativeLaunchTooLarge/g; $ft{'numeric_literal'} += s/\bcudaErrorLaunchFailure\b/hipErrorLaunchFailure/g; $ft{'numeric_literal'} += s/\bcudaErrorLaunchOutOfResources\b/hipErrorLaunchOutOfResources/g; $ft{'numeric_literal'} += s/\bcudaErrorLaunchTimeout\b/hipErrorLaunchTimeOut/g; @@ -1801,6 +1803,9 @@ sub simpleSubstitutions { $ft{'define'} += s/\bcudaTextureType3D\b/hipTextureType3D/g; $ft{'define'} += s/\bcudaTextureTypeCubemap\b/hipTextureTypeCubemap/g; $ft{'define'} += s/\bcudaTextureTypeCubemapLayered\b/hipTextureTypeCubemapLayered/g; + $ft{'define'} += s/\bcudaOccupancyDefault\b/hipOccupancyDefault/g; + $ft{'define'} += s/\bcudaCooperativeLaunchMultiDeviceNoPreSync\b/hipCooperativeLaunchMultiDeviceNoPreSync/g; + $ft{'define'} += s/\bcudaCooperativeLaunchMultiDeviceNoPostSync\b/hipCooperativeLaunchMultiDeviceNoPostSync/g; } # CUDA extern __shared__ syntax replace with HIP_DYNAMIC_SHARED() macro diff --git a/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md b/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md index ff14cf9bc1..5be6a4da8d 100644 --- a/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md +++ b/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md @@ -447,6 +447,7 @@ | 717 |*`CUDA_ERROR_INVALID_ADDRESS_SPACE`* | | | 718 |*`CUDA_ERROR_INVALID_PC`* | | | 719 |*`CUDA_ERROR_LAUNCH_FAILED`* |*`hipErrorLaunchFailure`* | +| 720 |*`CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE`* |*`hipErrorCooperativeLaunchTooLarge`* | | 800 |*`CUDA_ERROR_NOT_PERMITTED`* | | | 801 |*`CUDA_ERROR_NOT_SUPPORTED`* |*`hipErrorNotSupported`* | | 802 |*`CUDA_ERROR_SYSTEM_NOT_READY`* | | 10.0 | @@ -1063,8 +1064,8 @@ | `cuFuncSetSharedMemConfig` | | | `cuLaunchKernel` | `hipModuleLaunchKernel` | | `cuLaunchHostFunc` | | 10.0 | -| `cuLaunchCooperativeKernel` | | 9.0 | -| `cuLaunchCooperativeKernelMultiDevice` | | 9.0 | +| `cuLaunchCooperativeKernel` | `hipLaunchCooperativeKernel` | 9.0 | +| `cuLaunchCooperativeKernelMultiDevice` | `hipLaunchCooperativeKernelMultiDevice` | 9.0 | ## **19. Execution Control [DEPRECATED]** diff --git a/docs/markdown/CUDA_Runtime_API_functions_supported_by_HIP.md b/docs/markdown/CUDA_Runtime_API_functions_supported_by_HIP.md index 201d2aad0f..54e0c89e06 100644 --- a/docs/markdown/CUDA_Runtime_API_functions_supported_by_HIP.md +++ b/docs/markdown/CUDA_Runtime_API_functions_supported_by_HIP.md @@ -800,7 +800,7 @@ | 717 |*`cudaErrorInvalidAddressSpace`* | | | | 718 |*`cudaErrorInvalidPc`* | | | | 719 |*`cudaErrorLaunchFailure`* | |*`hipErrorLaunchFailure`* | -| 720 |*`cudaErrorCooperativeLaunchTooLarge`* | 9.0 | | +| 720 |*`cudaErrorCooperativeLaunchTooLarge`* | 9.0 |*`hipErrorCooperativeLaunchTooLarge`* | | 800 |*`cudaErrorNotPermitted`* | | | | 801 |*`cudaErrorNotSupported`* | |*`hipErrorNotSupported`* | | 802 |*`cudaErrorSystemNotReady`* | 10.0 | | @@ -1045,7 +1045,7 @@ | define |`cudaMemAttachGlobal` | |`hipMemAttachGlobal` | | define |`cudaMemAttachHost` | |`hipMemAttachHost` | | define |`cudaMemAttachSingle` | | | -| define |`cudaOccupancyDefault` | | | +| define |`cudaOccupancyDefault` | |`hipOccupancyDefault` | | define |`cudaOccupancyDisableCachingOverride` | | | | define |`cudaPeerAccessDefault` | | | | define |`cudaStreamDefault` | |`hipStreamDefault` | diff --git a/hipify-clang/src/CUDA2HIP_Driver_API_types.cpp b/hipify-clang/src/CUDA2HIP_Driver_API_types.cpp index 1c3c2634f7..a4d419384d 100644 --- a/hipify-clang/src/CUDA2HIP_Driver_API_types.cpp +++ b/hipify-clang/src/CUDA2HIP_Driver_API_types.cpp @@ -1284,6 +1284,8 @@ const std::map CUDA_DRIVER_TYPE_NAME_MAP{ {"CUDA_ERROR_INVALID_PC", {"hipErrorInvalidPc", "", CONV_NUMERIC_LITERAL, API_DRIVER, HIP_UNSUPPORTED}}, // 718 // cudaErrorLaunchFailure {"CUDA_ERROR_LAUNCH_FAILED", {"hipErrorLaunchFailure", "", CONV_NUMERIC_LITERAL, API_DRIVER}}, // 719 + // cudaErrorCooperativeLaunchTooLarge + {"CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE", {"hipErrorCooperativeLaunchTooLarge", "", CONV_NUMERIC_LITERAL, API_DRIVER, HIP_UNSUPPORTED}}, // 720 // cudaErrorNotPermitted {"CUDA_ERROR_NOT_PERMITTED", {"hipErrorNotPermitted", "", CONV_NUMERIC_LITERAL, API_DRIVER, HIP_UNSUPPORTED}}, // 800 // cudaErrorNotSupported diff --git a/hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp b/hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp index 5993f5d770..ade8b105f1 100644 --- a/hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp +++ b/hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp @@ -787,7 +787,7 @@ const std::map CUDA_RUNTIME_TYPE_NAME_MAP { {"cudaErrorInvalidPc", {"hipErrorInvalidPc", "", CONV_NUMERIC_LITERAL, API_RUNTIME, HIP_UNSUPPORTED}}, // 718 // CUDA_ERROR_LAUNCH_FAILED {"cudaErrorLaunchFailure", {"hipErrorLaunchFailure", "", CONV_NUMERIC_LITERAL, API_RUNTIME}}, // 719 - // no analogue + // CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE {"cudaErrorCooperativeLaunchTooLarge", {"hipErrorCooperativeLaunchTooLarge", "", CONV_NUMERIC_LITERAL, API_RUNTIME, HIP_UNSUPPORTED}}, // 720 // CUDA_ERROR_NOT_PERMITTED {"cudaErrorNotPermitted", {"hipErrorNotPermitted", "", CONV_NUMERIC_LITERAL, API_RUNTIME, HIP_UNSUPPORTED}}, // 800 diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 659a6c3c3a..8ee44be6ac 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -212,6 +212,11 @@ enum hipLimit_t { #define hipArrayCubemap 0x04 #define hipArrayTextureGather 0x08 +#define hipOccupancyDefault 0x00 + +#define hipCooperativeLaunchMultiDeviceNoPreSync 0x01 +#define hipCooperativeLaunchMultiDeviceNoPostSync 0x02 + /* * @brief hipJitOption * @enum @@ -2903,7 +2908,7 @@ hipError_t hipModuleLaunchKernel(hipFunction_t f, unsigned int gridDimX, unsigne * @param [in] stream Stream where the kernel should be dispatched. May be 0, in which case th * default stream is used with associated synchronization rules. * - * @returns hipSuccess, hipInvalidDevice, hipErrorNotInitialized, hipErrorInvalidValue + * @returns hipSuccess, hipInvalidDevice, hipErrorNotInitialized, hipErrorInvalidValue, hipErrorCooperativeLaunchTooLarge */ hipError_t hipLaunchCooperativeKernel(const void* f, dim3 gridDim, dim3 blockDimX, void** kernelParams, unsigned int sharedMemBytes, @@ -2917,7 +2922,7 @@ hipError_t hipLaunchCooperativeKernel(const void* f, dim3 gridDim, dim3 blockDim * @param [in] numDevices Size of the launchParamsList array. * @param [in] flags Flags to control launch behavior. * - * @returns hipSuccess, hipInvalidDevice, hipErrorNotInitialized, hipErrorInvalidValue + * @returns hipSuccess, hipInvalidDevice, hipErrorNotInitialized, hipErrorInvalidValue, hipErrorCooperativeLaunchTooLarge */ hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList, int numDevices, unsigned int flags); @@ -2960,7 +2965,7 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor( * @param [in] flags Extra flags for occupancy calculation (currently ignored) */ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( - int* numBlocks, const void* f, int blockSize, size_t dynSharedMemPerBlk, unsigned int flags); + int* numBlocks, const void* f, int blockSize, size_t dynSharedMemPerBlk, unsigned int flags __dparm(hipOccupancyDefault)); #if __HIP_VDI__ && !defined(__HCC__) /** diff --git a/include/hip/hip_runtime_api.h b/include/hip/hip_runtime_api.h index 64b2a85d8a..025688e98c 100644 --- a/include/hip/hip_runtime_api.h +++ b/include/hip/hip_runtime_api.h @@ -255,6 +255,10 @@ typedef enum __HIP_NODISCARD hipError_t { 713, ///< Produced when trying to unlock a non-page-locked memory. hipErrorLaunchFailure = 719, ///< An exception occurred on the device while executing a kernel. + hipErrorCooperativeLaunchTooLarge = + 720, ///< This error indicates that the number of blocks launched per grid for a kernel + ///< that was launched via cooperative launch APIs exceeds the maximum number of + ///< allowed blocks for the current device hipErrorNotSupported = 801, ///< Produced when the hip API is not supported/implemented hipErrorUnknown = 999, //< Unknown error. // HSA Runtime Error Codes start here. diff --git a/include/hip/nvcc_detail/hip_runtime_api.h b/include/hip/nvcc_detail/hip_runtime_api.h index 05a162478a..1ee84b5057 100644 --- a/include/hip/nvcc_detail/hip_runtime_api.h +++ b/include/hip/nvcc_detail/hip_runtime_api.h @@ -140,6 +140,12 @@ typedef enum cudaChannelFormatKind hipChannelFormatKind; #define hipLimitMallocHeapSize cudaLimitMallocHeapSize #define hipIpcMemLazyEnablePeerAccess cudaIpcMemLazyEnablePeerAccess +#define hipOccupancyDefault cudaOccupancyDefault + +#define hipCooperativeLaunchMultiDeviceNoPreSync cudaCooperativeLaunchMultiDeviceNoPreSync +#define hipCooperativeLaunchMultiDeviceNoPostSync cudaCooperativeLaunchMultiDeviceNoPostSync + + // enum CUjit_option redefines #define hipJitOptionMaxRegisters CU_JIT_MAX_REGISTERS #define hipJitOptionThreadsPerBlock CU_JIT_THREADS_PER_BLOCK @@ -267,6 +273,8 @@ inline static hipError_t hipCUDAErrorTohipError(cudaError_t cuError) { return hipErrorNotInitialized; case cudaErrorLaunchFailure: return hipErrorLaunchFailure; + case cudaErrorCooperativeLaunchTooLarge: + return hipErrorCooperativeLaunchTooLarge; case cudaErrorPriorLaunchFailure: return hipErrorPriorLaunchFailure; case cudaErrorLaunchOutOfResources: @@ -445,6 +453,8 @@ inline static hipError_t hipCUResultTohipError(CUresult cuError) { return hipErrorHostMemoryNotRegistered; case CUDA_ERROR_LAUNCH_FAILED: return hipErrorLaunchFailure; + case CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE: + return hipErrorCooperativeLaunchTooLarge; case CUDA_ERROR_NOT_SUPPORTED: return hipErrorNotSupported; case CUDA_ERROR_UNKNOWN: @@ -601,6 +611,8 @@ inline static cudaError_t hipErrorToCudaError(hipError_t hError) { return cudaErrorSetOnActiveProcess; case hipErrorLaunchFailure: return cudaErrorLaunchFailure; + case hipErrorCooperativeLaunchTooLarge: + return cudaErrorCooperativeLaunchTooLarge; case hipErrorNotSupported: return cudaErrorNotSupported; // HSA: does not exist in CUDA diff --git a/src/hip_hcc.cpp b/src/hip_hcc.cpp index 175d301ee1..85c08943bf 100644 --- a/src/hip_hcc.cpp +++ b/src/hip_hcc.cpp @@ -1796,6 +1796,8 @@ const char* ihipErrorString(hipError_t hip_error) { return "hipErrorMissingConfiguration"; case hipErrorLaunchFailure: return "hipErrorLaunchFailure"; + case hipErrorCooperativeLaunchTooLarge: + return "hipErrorCooperativeLaunchTooLarge"; case hipErrorPriorLaunchFailure: return "hipErrorPriorLaunchFailure"; case hipErrorLaunchTimeOut: From c8137263d62af5f0016fda855de74f335f3c43a2 Mon Sep 17 00:00:00 2001 From: Aryan Salmanpour Date: Fri, 31 Jan 2020 13:08:25 -0500 Subject: [PATCH 002/154] code clean up --- bin/hipify-perl | 5 ----- .../CUDA_Driver_API_functions_supported_by_HIP.md | 4 ++-- hipify-clang/src/CUDA2HIP_Driver_API_types.cpp | 8 ++++---- hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp | 8 ++++---- include/hip/nvcc_detail/hip_runtime_api.h | 6 ++++-- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/bin/hipify-perl b/bin/hipify-perl index d18e227c3d..4fe9b80349 100755 --- a/bin/hipify-perl +++ b/bin/hipify-perl @@ -1186,7 +1186,6 @@ sub simpleSubstitutions { $ft{'numeric_literal'} += s/\bCUDA_ERROR_INVALID_PTX\b/hipErrorInvalidKernelFile/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_INVALID_SOURCE\b/hipErrorInvalidSource/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_INVALID_VALUE\b/hipErrorInvalidValue/g; - $ft{'numeric_literal'} += s/\bCUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE\b/hipErrorCooperativeLaunchTooLarge/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_LAUNCH_FAILED\b/hipErrorLaunchFailure/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_LAUNCH_OUT_OF_RESOURCES\b/hipErrorLaunchOutOfResources/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_LAUNCH_TIMEOUT\b/hipErrorLaunchTimeOut/g; @@ -1660,7 +1659,6 @@ sub simpleSubstitutions { $ft{'numeric_literal'} += s/\bcudaErrorInvalidSource\b/hipErrorInvalidSource/g; $ft{'numeric_literal'} += s/\bcudaErrorInvalidSymbol\b/hipErrorInvalidSymbol/g; $ft{'numeric_literal'} += s/\bcudaErrorInvalidValue\b/hipErrorInvalidValue/g; - $ft{'numeric_literal'} += s/\bcudaErrorCooperativeLaunchTooLarge\b/hipErrorCooperativeLaunchTooLarge/g; $ft{'numeric_literal'} += s/\bcudaErrorLaunchFailure\b/hipErrorLaunchFailure/g; $ft{'numeric_literal'} += s/\bcudaErrorLaunchOutOfResources\b/hipErrorLaunchOutOfResources/g; $ft{'numeric_literal'} += s/\bcudaErrorLaunchTimeout\b/hipErrorLaunchTimeOut/g; @@ -1803,9 +1801,6 @@ sub simpleSubstitutions { $ft{'define'} += s/\bcudaTextureType3D\b/hipTextureType3D/g; $ft{'define'} += s/\bcudaTextureTypeCubemap\b/hipTextureTypeCubemap/g; $ft{'define'} += s/\bcudaTextureTypeCubemapLayered\b/hipTextureTypeCubemapLayered/g; - $ft{'define'} += s/\bcudaOccupancyDefault\b/hipOccupancyDefault/g; - $ft{'define'} += s/\bcudaCooperativeLaunchMultiDeviceNoPreSync\b/hipCooperativeLaunchMultiDeviceNoPreSync/g; - $ft{'define'} += s/\bcudaCooperativeLaunchMultiDeviceNoPostSync\b/hipCooperativeLaunchMultiDeviceNoPostSync/g; } # CUDA extern __shared__ syntax replace with HIP_DYNAMIC_SHARED() macro diff --git a/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md b/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md index 5be6a4da8d..8956533ed0 100644 --- a/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md +++ b/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md @@ -1064,8 +1064,8 @@ | `cuFuncSetSharedMemConfig` | | | `cuLaunchKernel` | `hipModuleLaunchKernel` | | `cuLaunchHostFunc` | | 10.0 | -| `cuLaunchCooperativeKernel` | `hipLaunchCooperativeKernel` | 9.0 | -| `cuLaunchCooperativeKernelMultiDevice` | `hipLaunchCooperativeKernelMultiDevice` | 9.0 | +| `cuLaunchCooperativeKernel` | | 9.0 | +| `cuLaunchCooperativeKernelMultiDevice` | | 9.0 | ## **19. Execution Control [DEPRECATED]** diff --git a/hipify-clang/src/CUDA2HIP_Driver_API_types.cpp b/hipify-clang/src/CUDA2HIP_Driver_API_types.cpp index a4d419384d..054de19800 100644 --- a/hipify-clang/src/CUDA2HIP_Driver_API_types.cpp +++ b/hipify-clang/src/CUDA2HIP_Driver_API_types.cpp @@ -1047,7 +1047,7 @@ const std::map CUDA_DRIVER_TYPE_NAME_MAP{ {"CUoccupancy_flags_enum", {"hipOccupancyFlags", "", CONV_TYPE, API_DRIVER, HIP_UNSUPPORTED}}, // CUoccupancy_flags enum values // cudaOccupancyDefault - {"CU_OCCUPANCY_DEFAULT", {"hipOccupancyDefault", "", CONV_NUMERIC_LITERAL, API_DRIVER, HIP_UNSUPPORTED}}, // 0x00 + {"CU_OCCUPANCY_DEFAULT", {"hipOccupancyDefault", "", CONV_NUMERIC_LITERAL, API_DRIVER}}, // 0x00 // cudaOccupancyDisableCachingOverride {"CU_OCCUPANCY_DISABLE_CACHING_OVERRIDE", {"hipOccupancyDisableCachingOverride", "", CONV_NUMERIC_LITERAL, API_DRIVER, HIP_UNSUPPORTED}}, // 0x01 @@ -1285,7 +1285,7 @@ const std::map CUDA_DRIVER_TYPE_NAME_MAP{ // cudaErrorLaunchFailure {"CUDA_ERROR_LAUNCH_FAILED", {"hipErrorLaunchFailure", "", CONV_NUMERIC_LITERAL, API_DRIVER}}, // 719 // cudaErrorCooperativeLaunchTooLarge - {"CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE", {"hipErrorCooperativeLaunchTooLarge", "", CONV_NUMERIC_LITERAL, API_DRIVER, HIP_UNSUPPORTED}}, // 720 + {"CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE", {"hipErrorCooperativeLaunchTooLarge", "", CONV_NUMERIC_LITERAL, API_DRIVER}}, // 720 // cudaErrorNotPermitted {"CUDA_ERROR_NOT_PERMITTED", {"hipErrorNotPermitted", "", CONV_NUMERIC_LITERAL, API_DRIVER, HIP_UNSUPPORTED}}, // 800 // cudaErrorNotSupported @@ -1600,9 +1600,9 @@ const std::map CUDA_DRIVER_TYPE_NAME_MAP{ // cudaArrayColorAttachment {"CUDA_ARRAY3D_COLOR_ATTACHMENT", {"hipArrayColorAttachment", "", CONV_DEFINE, API_DRIVER, HIP_UNSUPPORTED}}, // 0x20 // cudaCooperativeLaunchMultiDeviceNoPreSync - {"CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC", {"hipCooperativeLaunchMultiDeviceNoPreSync", "", CONV_DEFINE, API_DRIVER, HIP_UNSUPPORTED}}, // 0x01 + {"CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC", {"hipCooperativeLaunchMultiDeviceNoPreSync", "", CONV_DEFINE, API_DRIVER}}, // 0x01 // cudaCooperativeLaunchMultiDeviceNoPostSync - {"CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC", {"hipCooperativeLaunchMultiDeviceNoPostSync", "", CONV_DEFINE, API_DRIVER, HIP_UNSUPPORTED}}, // 0x02 + {"CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC", {"hipCooperativeLaunchMultiDeviceNoPostSync", "", CONV_DEFINE, API_DRIVER}}, // 0x02 // cudaExternalMemoryDedicated {"CUDA_EXTERNAL_MEMORY_DEDICATED", {"hipExternalMemoryDedicated", "", CONV_DEFINE, API_DRIVER, HIP_UNSUPPORTED}}, // 0x1 // cudaExternalSemaphoreSignalSkipNvSciBufMemSync diff --git a/hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp b/hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp index ade8b105f1..6eb9bfb2be 100644 --- a/hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp +++ b/hipify-clang/src/CUDA2HIP_Runtime_API_types.cpp @@ -788,7 +788,7 @@ const std::map CUDA_RUNTIME_TYPE_NAME_MAP { // CUDA_ERROR_LAUNCH_FAILED {"cudaErrorLaunchFailure", {"hipErrorLaunchFailure", "", CONV_NUMERIC_LITERAL, API_RUNTIME}}, // 719 // CUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE - {"cudaErrorCooperativeLaunchTooLarge", {"hipErrorCooperativeLaunchTooLarge", "", CONV_NUMERIC_LITERAL, API_RUNTIME, HIP_UNSUPPORTED}}, // 720 + {"cudaErrorCooperativeLaunchTooLarge", {"hipErrorCooperativeLaunchTooLarge", "", CONV_NUMERIC_LITERAL, API_RUNTIME}}, // 720 // CUDA_ERROR_NOT_PERMITTED {"cudaErrorNotPermitted", {"hipErrorNotPermitted", "", CONV_NUMERIC_LITERAL, API_RUNTIME, HIP_UNSUPPORTED}}, // 800 // CUDA_ERROR_NOT_SUPPORTED @@ -1327,9 +1327,9 @@ const std::map CUDA_RUNTIME_TYPE_NAME_MAP { // CUDA_ARRAY3D_COLOR_ATTACHMENT {"cudaArrayColorAttachment", {"hipArrayColorAttachment", "", CONV_DEFINE, API_RUNTIME, HIP_UNSUPPORTED}}, // 0x20 // CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC - {"cudaCooperativeLaunchMultiDeviceNoPreSync", {"hipCooperativeLaunchMultiDeviceNoPreSync", "", CONV_DEFINE, API_RUNTIME, HIP_UNSUPPORTED}}, // 0x01 + {"cudaCooperativeLaunchMultiDeviceNoPreSync", {"hipCooperativeLaunchMultiDeviceNoPreSync", "", CONV_DEFINE, API_RUNTIME}}, // 0x01 // CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC - {"cudaCooperativeLaunchMultiDeviceNoPostSync", {"hipCooperativeLaunchMultiDeviceNoPostSync", "", CONV_DEFINE, API_RUNTIME, HIP_UNSUPPORTED}}, // 0x02 + {"cudaCooperativeLaunchMultiDeviceNoPostSync", {"hipCooperativeLaunchMultiDeviceNoPostSync", "", CONV_DEFINE, API_RUNTIME}}, // 0x02 // CU_DEVICE_CPU ((CUdevice)-1) {"cudaCpuDeviceId", {"hipCpuDeviceId", "", CONV_DEFINE, API_RUNTIME, HIP_UNSUPPORTED}}, // ((int)-1) // CU_DEVICE_INVALID ((CUdevice)-2) @@ -1412,7 +1412,7 @@ const std::map CUDA_RUNTIME_TYPE_NAME_MAP { // no analogue {"cudaTextureTypeCubemapLayered", {"hipTextureTypeCubemapLayered", "", CONV_DEFINE, API_RUNTIME}}, // 0xFC // CU_OCCUPANCY_DEFAULT - {"cudaOccupancyDefault", {"hipOccupancyDefault", "", CONV_DEFINE, API_RUNTIME, HIP_UNSUPPORTED}}, // 0x00 + {"cudaOccupancyDefault", {"hipOccupancyDefault", "", CONV_DEFINE, API_RUNTIME}}, // 0x00 // CU_OCCUPANCY_DISABLE_CACHING_OVERRIDE {"cudaOccupancyDisableCachingOverride", {"hipOccupancyDisableCachingOverride", "", CONV_DEFINE, API_RUNTIME, HIP_UNSUPPORTED}}, // 0x01 // CU_STREAM_DEFAULT diff --git a/include/hip/nvcc_detail/hip_runtime_api.h b/include/hip/nvcc_detail/hip_runtime_api.h index 1ee84b5057..bcddce58b2 100644 --- a/include/hip/nvcc_detail/hip_runtime_api.h +++ b/include/hip/nvcc_detail/hip_runtime_api.h @@ -142,8 +142,10 @@ typedef enum cudaChannelFormatKind hipChannelFormatKind; #define hipOccupancyDefault cudaOccupancyDefault -#define hipCooperativeLaunchMultiDeviceNoPreSync cudaCooperativeLaunchMultiDeviceNoPreSync -#define hipCooperativeLaunchMultiDeviceNoPostSync cudaCooperativeLaunchMultiDeviceNoPostSync +#define hipCooperativeLaunchMultiDeviceNoPreSync + cudaCooperativeLaunchMultiDeviceNoPreSync +#define hipCooperativeLaunchMultiDeviceNoPostSync + cudaCooperativeLaunchMultiDeviceNoPostSync // enum CUjit_option redefines From 5c03a003c3ba6115ad06527bd509e7d3125cce5b Mon Sep 17 00:00:00 2001 From: Aryan Salmanpour Date: Fri, 31 Jan 2020 14:36:37 -0500 Subject: [PATCH 003/154] update the hipify-perl --- bin/hipify-perl | 8 ++++++++ 1 file changed, 8 insertions(+) mode change 100755 => 100644 bin/hipify-perl diff --git a/bin/hipify-perl b/bin/hipify-perl old mode 100755 new mode 100644 index 4fe9b80349..f6de5abae4 --- a/bin/hipify-perl +++ b/bin/hipify-perl @@ -1172,6 +1172,7 @@ sub simpleSubstitutions { $ft{'numeric_literal'} += s/\bCUDA_ERROR_ASSERT\b/hipErrorAssert/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_CONTEXT_ALREADY_CURRENT\b/hipErrorContextAlreadyCurrent/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_CONTEXT_ALREADY_IN_USE\b/hipErrorContextAlreadyInUse/g; + $ft{'numeric_literal'} += s/\bCUDA_ERROR_COOPERATIVE_LAUNCH_TOO_LARGE\b/hipErrorCooperativeLaunchTooLarge/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_DEINITIALIZED\b/hipErrorDeinitialized/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_ECC_UNCORRECTABLE\b/hipErrorECCNotCorrectable/g; $ft{'numeric_literal'} += s/\bCUDA_ERROR_FILE_NOT_FOUND\b/hipErrorFileNotFound/g; @@ -1528,6 +1529,7 @@ sub simpleSubstitutions { $ft{'numeric_literal'} += s/\bCU_MEMORYTYPE_UNIFIED\b/hipMemoryTypeUnified/g; $ft{'numeric_literal'} += s/\bCU_MEM_ATTACH_GLOBAL\b/hipMemAttachGlobal/g; $ft{'numeric_literal'} += s/\bCU_MEM_ATTACH_HOST\b/hipMemAttachHost/g; + $ft{'numeric_literal'} += s/\bCU_OCCUPANCY_DEFAULT\b/hipOccupancyDefault/g; $ft{'numeric_literal'} += s/\bCU_RESOURCE_TYPE_ARRAY\b/hipResourceTypeArray/g; $ft{'numeric_literal'} += s/\bCU_RESOURCE_TYPE_LINEAR\b/hipResourceTypeLinear/g; $ft{'numeric_literal'} += s/\bCU_RESOURCE_TYPE_MIPMAPPED_ARRAY\b/hipResourceTypeMipmappedArray/g; @@ -1636,6 +1638,7 @@ sub simpleSubstitutions { $ft{'numeric_literal'} += s/\bcudaErrorAlreadyMapped\b/hipErrorAlreadyMapped/g; $ft{'numeric_literal'} += s/\bcudaErrorArrayIsMapped\b/hipErrorArrayIsMapped/g; $ft{'numeric_literal'} += s/\bcudaErrorAssert\b/hipErrorAssert/g; + $ft{'numeric_literal'} += s/\bcudaErrorCooperativeLaunchTooLarge\b/hipErrorCooperativeLaunchTooLarge/g; $ft{'numeric_literal'} += s/\bcudaErrorCudartUnloading\b/hipErrorDeinitialized/g; $ft{'numeric_literal'} += s/\bcudaErrorDeviceAlreadyInUse\b/hipErrorContextAlreadyInUse/g; $ft{'numeric_literal'} += s/\bcudaErrorDeviceUninitialized\b/hipErrorInvalidContext/g; @@ -1749,6 +1752,8 @@ sub simpleSubstitutions { $ft{'define'} += s/\bCUDA_ARRAY3D_LAYERED\b/hipArrayLayered/g; $ft{'define'} += s/\bCUDA_ARRAY3D_SURFACE_LDST\b/hipArraySurfaceLoadStore/g; $ft{'define'} += s/\bCUDA_ARRAY3D_TEXTURE_GATHER\b/hipArrayTextureGather/g; + $ft{'define'} += s/\bCUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC\b/hipCooperativeLaunchMultiDeviceNoPostSync/g; + $ft{'define'} += s/\bCUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC\b/hipCooperativeLaunchMultiDeviceNoPreSync/g; $ft{'define'} += s/\bCU_LAUNCH_PARAM_BUFFER_POINTER\b/HIP_LAUNCH_PARAM_BUFFER_POINTER/g; $ft{'define'} += s/\bCU_LAUNCH_PARAM_BUFFER_SIZE\b/HIP_LAUNCH_PARAM_BUFFER_SIZE/g; $ft{'define'} += s/\bCU_LAUNCH_PARAM_END\b/HIP_LAUNCH_PARAM_END/g; @@ -1769,6 +1774,8 @@ sub simpleSubstitutions { $ft{'define'} += s/\bcudaArrayLayered\b/hipArrayLayered/g; $ft{'define'} += s/\bcudaArraySurfaceLoadStore\b/hipArraySurfaceLoadStore/g; $ft{'define'} += s/\bcudaArrayTextureGather\b/hipArrayTextureGather/g; + $ft{'define'} += s/\bcudaCooperativeLaunchMultiDeviceNoPostSync\b/hipCooperativeLaunchMultiDeviceNoPostSync/g; + $ft{'define'} += s/\bcudaCooperativeLaunchMultiDeviceNoPreSync\b/hipCooperativeLaunchMultiDeviceNoPreSync/g; $ft{'define'} += s/\bcudaDeviceBlockingSync\b/hipDeviceScheduleBlockingSync/g; $ft{'define'} += s/\bcudaDeviceLmemResizeToMax\b/hipDeviceLmemResizeToMax/g; $ft{'define'} += s/\bcudaDeviceMapHost\b/hipDeviceMapHost/g; @@ -1792,6 +1799,7 @@ sub simpleSubstitutions { $ft{'define'} += s/\bcudaIpcMemLazyEnablePeerAccess\b/hipIpcMemLazyEnablePeerAccess/g; $ft{'define'} += s/\bcudaMemAttachGlobal\b/hipMemAttachGlobal/g; $ft{'define'} += s/\bcudaMemAttachHost\b/hipMemAttachHost/g; + $ft{'define'} += s/\bcudaOccupancyDefault\b/hipOccupancyDefault/g; $ft{'define'} += s/\bcudaStreamDefault\b/hipStreamDefault/g; $ft{'define'} += s/\bcudaStreamNonBlocking\b/hipStreamNonBlocking/g; $ft{'define'} += s/\bcudaTextureType1D\b/hipTextureType1D/g; From 08123cff0d046350bc5a36ae13131e5c7c433252 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 6 Feb 2020 11:56:17 -0600 Subject: [PATCH 004/154] Use deque instead of vector for code readers so that the iterators and references will be stable --- src/program_state.inl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/program_state.inl b/src/program_state.inl index 272addd053..8df0f1cc0f 100644 --- a/src/program_state.inl +++ b/src/program_state.inl @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -202,7 +203,7 @@ public: std::function>; std::pair< std::mutex, - std::vector>> code_readers; + std::deque>> code_readers; program_state_impl() { // Create placeholder for each agent for the per-agent members. @@ -418,7 +419,7 @@ public: decltype(code_readers.second)::iterator it; { std::lock_guard lck{code_readers.first}; - it = code_readers.second.emplace(code_readers.second.end(), + it = code_readers.second.emplace_back(code_readers.second.end(), move(file), move(tmp)); } From f34f96350c6c937d06c0a828d5c3f6f745dcb896 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 6 Feb 2020 12:04:50 -0600 Subject: [PATCH 005/154] Fix compile error --- src/program_state.inl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/program_state.inl b/src/program_state.inl index 8df0f1cc0f..0442874900 100644 --- a/src/program_state.inl +++ b/src/program_state.inl @@ -419,8 +419,7 @@ public: decltype(code_readers.second)::iterator it; { std::lock_guard lck{code_readers.first}; - it = code_readers.second.emplace_back(code_readers.second.end(), - move(file), move(tmp)); + it = code_readers.second.emplace_back(move(file), move(tmp)); } auto check_hsa_error = [](hsa_status_t s) { From 85560d38b108735d5a2cc02a8b85bf235bcb0e54 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 6 Feb 2020 12:15:29 -0600 Subject: [PATCH 006/154] Assign the iterator --- src/program_state.inl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/program_state.inl b/src/program_state.inl index 0442874900..c8eb2f297c 100644 --- a/src/program_state.inl +++ b/src/program_state.inl @@ -419,7 +419,8 @@ public: decltype(code_readers.second)::iterator it; { std::lock_guard lck{code_readers.first}; - it = code_readers.second.emplace_back(move(file), move(tmp)); + code_readers.second.emplace_back(move(file), move(tmp)); + it = std::prev(code_readers.second.end()); } auto check_hsa_error = [](hsa_status_t s) { From 53614247021166b54556271c2fdc1b1fb0a9cba6 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 6 Feb 2020 16:21:40 -0600 Subject: [PATCH 007/154] Add multithreaded test --- .../module/hipModuleLoadDataMultThreaded.cpp | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp new file mode 100644 index 0000000000..f989a14a06 --- /dev/null +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -0,0 +1,144 @@ +/* +Copyright (c) 2015-2016 Advanced Micro Devices, Inc. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANNTY OF ANY KIND, EXPRESS OR +IMPLIED, INNCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANNY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/* HIT_START + * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS -std=c++11 + * TEST: %t + * HIT_END + */ + +#include "hip/hip_runtime.h" +#include "hip/hip_runtime_api.h" +#include +#include +#include +#include +#include + +#include "test_common.h" + +#define LEN 64 +#define SIZE LEN << 2 +#define THREADS 64 + +#define FILENAME "vcpy_kernel.code" +#define kernel_name "hello_world" + +using ModuleFunction = std::pair; + +ModuleFunction load() { + hipModule_t Module; + hipFunction_t Function; + std::ifstream file(FILENAME, std::ios::binary | std::ios::ate); + std::streamsize fsize = file.tellg(); + file.seekg(0, std::ios::beg); + + std::vector buffer(fsize); + if (file.read(buffer.data(), fsize)) { + HIPCHECK(hipModuleLoadData(&Module, &buffer[0])); + HIPCHECK(hipModuleGetFunction(&Function, Module, kernel_name)); + } + else { + failed("could not open code object '%s'\n", FILENAME); + } + return {Module, Function}; +} + +void run(ModuleFunction mf) { + hipModule_t Module = mf.first; + hipFunction_t Function = mf.second; + float *A, *B, *Ad, *Bd; + A = new float[LEN]; + B = new float[LEN]; + + for (uint32_t i = 0; i < LEN; i++) { + A[i] = i * 1.0f; + B[i] = 0.0f; + } + + HIPCHECK(hipMalloc((void**)&Ad, SIZE)); + HIPCHECK(hipMalloc((void**)&Bd, SIZE)); + + HIPCHECK(hipMemcpy(Ad, A, SIZE, hipMemcpyHostToDevice)); + HIPCHECK(hipMemcpy(Bd, B, SIZE, hipMemcpyHostToDevice)); + + hipStream_t stream; + HIPCHECK(hipStreamCreate(&stream)); + + struct { + void* _Ad; + void* _Bd; + } args; + args._Ad = (void*) Ad; + args._Bd = (void*) Bd; + size_t size = sizeof(args); + + void* config[] = {HIP_LAUNCH_PARAM_BUFFER_POINTER, &args, HIP_LAUNCH_PARAM_BUFFER_SIZE, &size, + HIP_LAUNCH_PARAM_END}; + HIPCHECK(hipModuleLaunchKernel(Function, 1, 1, 1, LEN, 1, 1, 0, stream, NULL, (void**)&config)); + + HIPCHECK(hipStreamDestroy(stream)); + + HIPCHECK(hipModuleUnload(Module)); + + HIPCHECK(hipMemcpy(B, Bd, SIZE, hipMemcpyDeviceToHost)); + + for (uint32_t i = 0; i < LEN; i++) { + assert(A[i] == B[i]); + } +} + +struct joinable_thread : std::thread +{ + template + joinable_thread(Xs&&... xs) : std::thread(std::forward(xs)...) // NOLINT + { + } + + joinable_thread& operator=(joinable_thread&& other) = default; + joinable_thread(joinable_thread&& other) = default; + + ~joinable_thread() + { + if(this->joinable()) + this->join(); + } +}; + +void run_multi_threads(uint32_t n) { + std::vector mf(n); + { + std::vector threads; + for (uint32_t i = 0; i < n; i++) { + threads.emplace_back(std::thread{[=, &mf] { + mf[i] = load(); + }}); + } + } + for(auto&& x:mf) + run(x); +} + +int main() { + + HIPCHECK(hipInit(0)); + run_multi_threads(THREADS); + + passed(); +} From 8e494cfce8c5f2b4de03b6c91bb26ba04a4c5155 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 6 Feb 2020 16:23:29 -0600 Subject: [PATCH 008/154] Make threads a multiple of hardware concurrency --- tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index f989a14a06..31e930086b 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -35,7 +35,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN << 2 -#define THREADS 64 +#define THREADS 4 #define FILENAME "vcpy_kernel.code" #define kernel_name "hello_world" @@ -138,7 +138,7 @@ void run_multi_threads(uint32_t n) { int main() { HIPCHECK(hipInit(0)); - run_multi_threads(THREADS); + run_multi_threads(THREADS * std::thread::hardware_concurrency()); passed(); } From d77ede7015f1e9e4dfb7685d3ee109e56d15ef5f Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Feb 2020 10:13:28 -0600 Subject: [PATCH 009/154] Output on failure --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b8bd24cd74..734e875e03 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -177,7 +177,7 @@ def docker_build_inside_image( def build_image, String inside_args, String platf cd ${build_dir_rel} make install -j\$(nproc) make build_tests -i -j\$(nproc) - ctest -E "(hipMultiThreadDevice-pyramid|hipMemoryAllocateCoherentDriver)" + ctest --output-on-failure -E "(hipMultiThreadDevice-pyramid|hipMemoryAllocateCoherentDriver)" """ // If unit tests output a junit or xunit file in the future, jenkins can parse that file // to display test results on the dashboard From 5a29f27455c15634ac4a95f4a09455b8824d4a0d Mon Sep 17 00:00:00 2001 From: Aryan Salmanpour Date: Mon, 10 Feb 2020 11:44:40 -0500 Subject: [PATCH 010/154] Fix a typo causing a build error --- include/hip/hcc_detail/hip_runtime_api.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 1636a782a0..76209ef6a7 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -2964,6 +2964,7 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor( * @param [in] dynSharedMemPerBlk dynamic shared memory usage (in bytes) intended for each block * @param [in] flags Extra flags for occupancy calculation (currently ignored) */ +hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( uint32_t* numBlocks, hipFunction_t f, uint32_t blockSize, size_t dynSharedMemPerBlk, unsigned int flags __dparm(hipOccupancyDefault)); #if __HIP_VDI__ && !defined(__HCC__) From 29a257d79b48d65906a1bf3b1b314ced13af61ee Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 10 Feb 2020 13:37:45 -0600 Subject: [PATCH 011/154] Add setDevice to try and initialize the context on cuda --- tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 31e930086b..ff2d5b1ef3 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -127,6 +127,7 @@ void run_multi_threads(uint32_t n) { std::vector threads; for (uint32_t i = 0; i < n; i++) { threads.emplace_back(std::thread{[=, &mf] { + hipSetDevice(0); mf[i] = load(); }}); } From e5d077f70e27540b626cf1aedaf73c278343a685 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 10 Feb 2020 15:52:34 -0600 Subject: [PATCH 012/154] Create context for cuda --- .../module/hipModuleLoadDataMultThreaded.cpp | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index ff2d5b1ef3..92a1cb77c7 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -35,28 +35,31 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN << 2 -#define THREADS 4 +#define THREADS 8 #define FILENAME "vcpy_kernel.code" #define kernel_name "hello_world" using ModuleFunction = std::pair; -ModuleFunction load() { - hipModule_t Module; - hipFunction_t Function; +std::vector load_file() +{ std::ifstream file(FILENAME, std::ios::binary | std::ios::ate); std::streamsize fsize = file.tellg(); file.seekg(0, std::ios::beg); std::vector buffer(fsize); - if (file.read(buffer.data(), fsize)) { - HIPCHECK(hipModuleLoadData(&Module, &buffer[0])); - HIPCHECK(hipModuleGetFunction(&Function, Module, kernel_name)); - } - else { + if (!file.read(buffer.data(), fsize)) { failed("could not open code object '%s'\n", FILENAME); } + return buffer; +} + +ModuleFunction load(const std::vector& buffer) { + hipModule_t Module; + hipFunction_t Function; + HIPCHECK(hipModuleLoadData(&Module, &buffer[0])); + HIPCHECK(hipModuleGetFunction(&Function, Module, kernel_name)); return {Module, Function}; } @@ -121,25 +124,37 @@ struct joinable_thread : std::thread } }; +hipCtx_t create_context() { + hipDevice_t device; + HIPCHECK(hipDeviceGet(&device, 0)); + + hipCtx_t ctx; + HIPCHECK(hipCtxCreate(&ctx, 0, device)); + return ctx; +} + void run_multi_threads(uint32_t n) { std::vector mf(n); { + auto buffer = load_file(); std::vector threads; for (uint32_t i = 0; i < n; i++) { - threads.emplace_back(std::thread{[=, &mf] { - hipSetDevice(0); - mf[i] = load(); + threads.emplace_back(std::thread{[&, i, buffer] { + mf[i] = load(buffer); }}); } } for(auto&& x:mf) run(x); + } int main() { HIPCHECK(hipInit(0)); + auto ctx = create_context(); run_multi_threads(THREADS * std::thread::hardware_concurrency()); + hipCtxDestroy(ctx); passed(); } From 2d9a2d866ca516441ffd07182c27c3d9e07237e1 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 10 Feb 2020 16:01:53 -0600 Subject: [PATCH 013/154] Set context on each thread --- .../src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 92a1cb77c7..cdb4c81c94 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -134,27 +134,27 @@ hipCtx_t create_context() { } void run_multi_threads(uint32_t n) { + auto ctx = create_context(); std::vector mf(n); { auto buffer = load_file(); std::vector threads; for (uint32_t i = 0; i < n; i++) { threads.emplace_back(std::thread{[&, i, buffer] { + HIPCHECK(hipCtxSetCurrent(ctx)); mf[i] = load(buffer); }}); } } for(auto&& x:mf) run(x); - + hipCtxDestroy(ctx); } int main() { HIPCHECK(hipInit(0)); - auto ctx = create_context(); run_multi_threads(THREADS * std::thread::hardware_concurrency()); - hipCtxDestroy(ctx); passed(); } From 30e8dfdd8637fd5e6282a4178d96f6044a5a6cdd Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 10 Feb 2020 16:37:34 -0600 Subject: [PATCH 014/154] Reduce threads on cuda --- tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index cdb4c81c94..3b22176927 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -35,7 +35,11 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN << 2 +#ifdef __CUDACC__ +#define THREADS 1 +#else #define THREADS 8 +#endif #define FILENAME "vcpy_kernel.code" #define kernel_name "hello_world" From b9f97ec3feb7c9fd6c70a1ea4c7833d0956c97a9 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 10 Feb 2020 17:23:58 -0600 Subject: [PATCH 015/154] Skip test on cuda --- .../module/hipModuleLoadDataMultThreaded.cpp | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 3b22176927..03a2b82b8b 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS -std=c++11 + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ @@ -35,11 +35,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN << 2 -#ifdef __CUDACC__ -#define THREADS 1 -#else #define THREADS 8 -#endif #define FILENAME "vcpy_kernel.code" #define kernel_name "hello_world" @@ -128,31 +124,19 @@ struct joinable_thread : std::thread } }; -hipCtx_t create_context() { - hipDevice_t device; - HIPCHECK(hipDeviceGet(&device, 0)); - - hipCtx_t ctx; - HIPCHECK(hipCtxCreate(&ctx, 0, device)); - return ctx; -} - void run_multi_threads(uint32_t n) { - auto ctx = create_context(); std::vector mf(n); { auto buffer = load_file(); std::vector threads; for (uint32_t i = 0; i < n; i++) { threads.emplace_back(std::thread{[&, i, buffer] { - HIPCHECK(hipCtxSetCurrent(ctx)); mf[i] = load(buffer); }}); } } for(auto&& x:mf) run(x); - hipCtxDestroy(ctx); } int main() { From 959f1b0f0e6dd1da478920940bdbd590d5c06a4b Mon Sep 17 00:00:00 2001 From: Aryan Salmanpour Date: Tue, 11 Feb 2020 12:16:51 -0500 Subject: [PATCH 016/154] fix build error in nvcc path --- include/hip/nvcc_detail/hip_runtime_api.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/hip/nvcc_detail/hip_runtime_api.h b/include/hip/nvcc_detail/hip_runtime_api.h index bcddce58b2..6e0d02d0c0 100644 --- a/include/hip/nvcc_detail/hip_runtime_api.h +++ b/include/hip/nvcc_detail/hip_runtime_api.h @@ -142,9 +142,9 @@ typedef enum cudaChannelFormatKind hipChannelFormatKind; #define hipOccupancyDefault cudaOccupancyDefault -#define hipCooperativeLaunchMultiDeviceNoPreSync +#define hipCooperativeLaunchMultiDeviceNoPreSync \ cudaCooperativeLaunchMultiDeviceNoPreSync -#define hipCooperativeLaunchMultiDeviceNoPostSync +#define hipCooperativeLaunchMultiDeviceNoPostSync \ cudaCooperativeLaunchMultiDeviceNoPostSync From fd98514113a3530886bcb42d7c61ddc42f5df70b Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 11 Feb 2020 11:26:24 -0600 Subject: [PATCH 017/154] Try to initialize the primary context on cuda --- .../runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 03a2b82b8b..6612392064 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc + * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS -std=c++11 * TEST: %t * HIT_END */ @@ -127,10 +127,16 @@ struct joinable_thread : std::thread void run_multi_threads(uint32_t n) { std::vector mf(n); { + hipDevice_t device; + HIPCHECK(hipDeviceGet(&device, 0)); + auto buffer = load_file(); std::vector threads; for (uint32_t i = 0; i < n; i++) { threads.emplace_back(std::thread{[&, i, buffer] { + hipCtx_t ctx; + HIPCHECK(hipDevicePrimaryCtxRetain(&ctx, device)); + mf[i] = load(buffer); }}); } From bff8cbe950058e51eb911d1a9dea01c97ce6aeda Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 11 Feb 2020 11:46:29 -0600 Subject: [PATCH 018/154] Push ctx to the stack as current --- tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 6612392064..09fedc3660 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -136,7 +136,7 @@ void run_multi_threads(uint32_t n) { threads.emplace_back(std::thread{[&, i, buffer] { hipCtx_t ctx; HIPCHECK(hipDevicePrimaryCtxRetain(&ctx, device)); - + HIPCHECK(hipCtxPushCurrent(ctx)); mf[i] = load(buffer); }}); } From e82e3c2339f1245818b32ae496672564997e4f88 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 11 Feb 2020 12:34:10 -0600 Subject: [PATCH 019/154] Revert "Push ctx to the stack as current" This reverts commit bff8cbe950058e51eb911d1a9dea01c97ce6aeda. --- tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 09fedc3660..6612392064 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -136,7 +136,7 @@ void run_multi_threads(uint32_t n) { threads.emplace_back(std::thread{[&, i, buffer] { hipCtx_t ctx; HIPCHECK(hipDevicePrimaryCtxRetain(&ctx, device)); - HIPCHECK(hipCtxPushCurrent(ctx)); + mf[i] = load(buffer); }}); } From 26bb6a97a7413897004ffa771496c2930488a1a6 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 11 Feb 2020 12:34:11 -0600 Subject: [PATCH 020/154] Revert "Try to initialize the primary context on cuda" This reverts commit fd98514113a3530886bcb42d7c61ddc42f5df70b. --- .../runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 6612392064..03a2b82b8b 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS -std=c++11 + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ @@ -127,16 +127,10 @@ struct joinable_thread : std::thread void run_multi_threads(uint32_t n) { std::vector mf(n); { - hipDevice_t device; - HIPCHECK(hipDeviceGet(&device, 0)); - auto buffer = load_file(); std::vector threads; for (uint32_t i = 0; i < n; i++) { threads.emplace_back(std::thread{[&, i, buffer] { - hipCtx_t ctx; - HIPCHECK(hipDevicePrimaryCtxRetain(&ctx, device)); - mf[i] = load(buffer); }}); } From 8c6934223bffe4b030e18c15868107cc8d8941d2 Mon Sep 17 00:00:00 2001 From: ansurya <50609411+ansurya@users.noreply.github.com> Date: Thu, 13 Feb 2020 14:21:51 +0530 Subject: [PATCH 021/154] Reduce GPU copying based on arch it runs on (#1751) Implements SWDEV-213230. --- CMakeLists.txt | 2 +- include/hip/hcc_detail/code_object_bundle.hpp | 20 +++++++++++-------- lpl_ca/CMakeLists.txt | 4 ++-- src/hip_hcc.cpp | 11 ++++++++++ src/program_state.inl | 3 ++- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3eedf35b09..3a77735865 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -336,9 +336,9 @@ if(HIP_PLATFORM STREQUAL "hcc") target_link_libraries(hip_hcc_static PRIVATE hc_am) add_library(hiprtc SHARED src/hiprtc.cpp src/code_object_bundle.cpp) + target_compile_options(hiprtc PRIVATE -DDISABLE_REDUCED_GPU_BLOB_COPY) set_property ( TARGET hiprtc PROPERTY VERSION "${HIP_LIB_VERSION_STRING}" ) set_property ( TARGET hiprtc PROPERTY SOVERSION "${HIP_LIB_VERSION_MAJOR}" ) - target_include_directories( hiprtc SYSTEM PRIVATE ${PROJECT_SOURCE_DIR}/include ${HSA_PATH}/include) diff --git a/include/hip/hcc_detail/code_object_bundle.hpp b/include/hip/hcc_detail/code_object_bundle.hpp index f312d2e79b..77e0d706d6 100644 --- a/include/hip/hcc_detail/code_object_bundle.hpp +++ b/include/hip/hcc_detail/code_object_bundle.hpp @@ -31,9 +31,11 @@ THE SOFTWARE. #include #include #include - +#include namespace hip_impl { - +#if !defined(DISABLE_REDUCED_GPU_BLOB_COPY) +std::unordered_set& get_all_gpuarch(); +#endif inline std::string transmogrify_triple(const std::string& triple) { @@ -43,7 +45,6 @@ std::string transmogrify_triple(const std::string& triple) if (triple.find(old_prefix) == 0) { return new_prefix + triple.substr(sizeof(old_prefix) - 1); } - return (triple.find(new_prefix) == 0) ? triple : ""; } @@ -114,9 +115,7 @@ class Bundled_code_header { friend inline bool read(RandomAccessIterator f, RandomAccessIterator l, Bundled_code_header& x) { if (f == l) return false; - std::copy_n(f, sizeof(x.header_.cbuf_), x.header_.cbuf_); - if (valid(x)) { x.bundles_.resize(x.header_.bundle_cnt_); @@ -126,11 +125,16 @@ class Bundled_code_header { it += sizeof(y.header.cbuf); y.triple.assign(it, it + y.header.triple_sz); - + #ifdef DISABLE_REDUCED_GPU_BLOB_COPY std::copy_n(f + y.header.offset, y.header.bundle_sz, std::back_inserter(y.blob)); - + #else + auto& gpuArch = get_all_gpuarch(); + auto itgpuArch = std::find(gpuArch.begin(),gpuArch.end(),y.triple); + if (itgpuArch != gpuArch.end()){ + std::copy_n(f + y.header.offset, y.header.bundle_sz, std::back_inserter(y.blob)); + } + #endif it += y.header.triple_sz; - x.bundled_code_size = std::max(x.bundled_code_size, y.header.offset + y.header.bundle_sz); } diff --git a/lpl_ca/CMakeLists.txt b/lpl_ca/CMakeLists.txt index b36d73bbcb..ac01a6a0ab 100644 --- a/lpl_ca/CMakeLists.txt +++ b/lpl_ca/CMakeLists.txt @@ -26,7 +26,7 @@ target_include_directories(ca PUBLIC ${PROJECT_SOURCE_DIR}/src) find_library( hsart NAMES libhsa-runtime64.so libhsa-runtime64.so.1 HINTS ${HSA_PATH}/lib) target_link_libraries(ca PUBLIC ${hsart}) -target_compile_options(ca PUBLIC -Wall) +target_compile_options(ca PUBLIC -DDISABLE_REDUCED_GPU_BLOB_COPY -Wall) install(TARGETS ca RUNTIME DESTINATION bin) -#-------------------------------------CA---------------------------------------# \ No newline at end of file +#-------------------------------------CA---------------------------------------# diff --git a/src/hip_hcc.cpp b/src/hip_hcc.cpp index 85c08943bf..af40b29ea9 100644 --- a/src/hip_hcc.cpp +++ b/src/hip_hcc.cpp @@ -39,6 +39,7 @@ THE SOFTWARE. #include #include #include +#include #include #include @@ -2542,6 +2543,16 @@ hipError_t hipHccGetAcceleratorView(hipStream_t stream, hc::accelerator_view** a // TODO - add a contect sequence number for debug. Print operator<< ctx:0.1 (device.ctx) namespace hip_impl { + std::unordered_set& get_all_gpuarch() { + static std::unordered_set r{}; + static std::once_flag init; + std::call_once(init, []() { + for (int i=0; i < g_deviceCnt; i++){ + r.insert("hcc-amdgcn-amd-amdhsa--gfx"+std::to_string(g_deviceArray[i]->_props.gcnArch)); + }}); + return r; + } + std::vector all_hsa_agents() { std::vector r{}; std::vector visible_accelerators; diff --git a/src/program_state.inl b/src/program_state.inl index 272addd053..3581dea229 100644 --- a/src/program_state.inl +++ b/src/program_state.inl @@ -244,7 +244,8 @@ public: if (!valid(tmp)) break; for (auto&& bundle : bundles(tmp)) { - impl.code_object_blobs.second[elf][triple_to_hsa_isa(bundle.triple)].push_back(bundle.blob); + if(bundle.blob.size()) + impl.code_object_blobs.second[elf][triple_to_hsa_isa(bundle.triple)].push_back(bundle.blob); } blob_it += tmp.bundled_code_size; From fc5256fd28c80eceef85758a63260c1ef0cf189e Mon Sep 17 00:00:00 2001 From: Sarbojit2019 <52527887+SarbojitAMD@users.noreply.github.com> Date: Thu, 13 Feb 2020 14:22:11 +0530 Subject: [PATCH 022/154] ihipEnablePeerAccess return error if peer is not accessible (#1858) hipDeviceEnablePeerAccess returns success and adds peer into the list even if it is not accessible which creates problem in hipMalloc when it tries to share the ptr to peer device. Proposed change is to check the access status before updating the peer list and update only when it can access the peer. --- src/hip_peer.cpp | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/hip_peer.cpp b/src/hip_peer.cpp index 7781af1dbe..8fd66a52bb 100644 --- a/src/hip_peer.cpp +++ b/src/hip_peer.cpp @@ -128,17 +128,24 @@ hipError_t ihipEnablePeerAccess(TlsData* tls, hipCtx_t peerCtx, unsigned int fla if (thisCtx == peerCtx) { err = hipErrorInvalidDevice; // Can't enable peer access to self. } else if ((thisCtx != NULL) && (peerCtx != NULL)) { - LockedAccessor_CtxCrit_t peerCrit(peerCtx->criticalData()); - // Add thisCtx to peerCtx's access list so that new allocations on peer will be made - // visible to this device: - bool isNewPeer = peerCrit->addPeerWatcher(peerCtx, thisCtx); - if (isNewPeer) { - tprintf(DB_MEM, "device=%s can now see all memory allocated on peer=%s\n", - thisCtx->toString().c_str(), peerCtx->toString().c_str()); - am_memtracker_update_peers(peerCtx->getDevice()->_acc, peerCrit->peerCnt(), - peerCrit->peerAgents()); + + int canAccess = 0; + if ((hipSuccess != ihipDeviceCanAccessPeer(&canAccess,thisCtx,peerCtx)) || (canAccess == 0)){ + tprintf(DB_MEM, "device=%s can't access peer=%s\n",thisCtx->toString().c_str(), peerCtx->toString().c_str()); + err = hipErrorInvalidDevice; } else { - err = hipErrorPeerAccessAlreadyEnabled; + LockedAccessor_CtxCrit_t peerCrit(peerCtx->criticalData()); + // Add thisCtx to peerCtx's access list so that new allocations on peer will be made + // visible to this device: + bool isNewPeer = peerCrit->addPeerWatcher(peerCtx, thisCtx); + if (isNewPeer) { + tprintf(DB_MEM, "device=%s can now see all memory allocated on peer=%s\n", + thisCtx->toString().c_str(), peerCtx->toString().c_str()); + am_memtracker_update_peers(peerCtx->getDevice()->_acc, peerCrit->peerCnt(), + peerCrit->peerAgents()); + } else { + err = hipErrorPeerAccessAlreadyEnabled; + } } } else { err = hipErrorInvalidDevice; From 1109cbff83ade23e54e3f63d9554117d7314ee99 Mon Sep 17 00:00:00 2001 From: Sarbojit2019 <52527887+SarbojitAMD@users.noreply.github.com> Date: Thu, 13 Feb 2020 14:22:46 +0530 Subject: [PATCH 023/154] [hip] Fix for bug introduced in #1770 when blockSize is non-power of 2 (#1864) Fixes SWDEV-222161 --- src/hip_module.cpp | 95 +++++++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/src/hip_module.cpp b/src/hip_module.cpp index 16606e8016..44f0f108a6 100644 --- a/src/hip_module.cpp +++ b/src/hip_module.cpp @@ -134,10 +134,10 @@ extern hipError_t ihipGetDeviceProperties(hipDeviceProp_t* props, int device); return ihipLogStatus(hipStatus); \ } -hipError_t ihipModuleLaunchKernel(TlsData *tls, hipFunction_t f, uint32_t gridSizeX, - uint32_t gridSizeY, uint32_t gridSizeZ, - uint32_t blockSizeX, uint32_t blockSizeY, - uint32_t blockSizeZ, size_t sharedMemBytes, +hipError_t ihipModuleLaunchKernel(TlsData *tls, hipFunction_t f, uint32_t globalWorkSizeX, + uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ, + uint32_t localWorkSizeX, uint32_t localWorkSizeY, + uint32_t localWorkSizeZ, size_t sharedMemBytes, hipStream_t hStream, void** kernelParams, void** extra, hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags, bool isStreamLocked = 0, void** impCoopParams = 0) { @@ -146,14 +146,6 @@ hipError_t ihipModuleLaunchKernel(TlsData *tls, hipFunction_t f, uint32_t gridSi auto ctx = ihipGetTlsDefaultCtx(); hipError_t ret = hipSuccess; - size_t globalWorkSizeX = (size_t)gridSizeX * (size_t)blockSizeX; - size_t globalWorkSizeY = (size_t)gridSizeY * (size_t)blockSizeY; - size_t globalWorkSizeZ = (size_t)gridSizeZ * (size_t)blockSizeZ; - if(globalWorkSizeX > UINT32_MAX || globalWorkSizeY > UINT32_MAX || globalWorkSizeZ > UINT32_MAX) - { - return hipErrorInvalidConfiguration; - } - if (ctx == nullptr) { ret = hipErrorInvalidDevice; @@ -211,8 +203,8 @@ hipError_t ihipModuleLaunchKernel(TlsData *tls, hipFunction_t f, uint32_t gridSi lp.dynamic_group_mem_bytes = sharedMemBytes; // TODO - this should be part of preLaunchKernel. hStream = ihipPreLaunchKernel( - hStream, dim3(globalWorkSizeX/blockSizeX, globalWorkSizeY/blockSizeY, globalWorkSizeZ/blockSizeZ), - dim3(blockSizeX, blockSizeY, blockSizeZ), &lp, f->_name.c_str(), isStreamLocked); + hStream, dim3(globalWorkSizeX/localWorkSizeX, globalWorkSizeY/localWorkSizeY, globalWorkSizeZ/localWorkSizeZ), + dim3(localWorkSizeX, localWorkSizeY, localWorkSizeZ), &lp, f->_name.c_str(), isStreamLocked); hsa_kernel_dispatch_packet_t aql; @@ -221,9 +213,9 @@ hipError_t ihipModuleLaunchKernel(TlsData *tls, hipFunction_t f, uint32_t gridSi // aql.completion_signal._handle = 0; // aql.kernarg_address = 0; - aql.workgroup_size_x = blockSizeX; - aql.workgroup_size_y = blockSizeY; - aql.workgroup_size_z = blockSizeZ; + aql.workgroup_size_x = localWorkSizeX; + aql.workgroup_size_y = localWorkSizeY; + aql.workgroup_size_z = localWorkSizeZ; aql.grid_size_x = globalWorkSizeX; aql.grid_size_y = globalWorkSizeY; aql.grid_size_z = globalWorkSizeZ; @@ -283,8 +275,17 @@ hipError_t hipModuleLaunchKernel(hipFunction_t f, uint32_t gridDimX, uint32_t gr void** kernelParams, void** extra) { HIP_INIT_API(hipModuleLaunchKernel, f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra); + + size_t globalWorkSizeX = (size_t)gridDimX * (size_t)blockDimX; + size_t globalWorkSizeY = (size_t)gridDimY * (size_t)blockDimY; + size_t globalWorkSizeZ = (size_t)gridDimZ * (size_t)blockDimZ; + if(globalWorkSizeX > UINT32_MAX || globalWorkSizeY > UINT32_MAX || globalWorkSizeZ > UINT32_MAX) + { + return hipErrorInvalidConfiguration; + } + return ihipLogStatus(ihipModuleLaunchKernel(tls, - f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, + f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra, nullptr, nullptr, 0)); } @@ -297,11 +298,8 @@ hipError_t hipExtModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX, HIP_INIT_API(hipExtModuleLaunchKernel, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, localWorkSizeX, localWorkSizeY, localWorkSizeZ, sharedMemBytes, hStream, kernelParams, extra); - if(localWorkSizeX == 0 || localWorkSizeY == 0 || localWorkSizeZ == 0) - return hipErrorInvalidValue; - return ihipLogStatus(ihipModuleLaunchKernel(tls, - f, globalWorkSizeX/localWorkSizeX, globalWorkSizeY/localWorkSizeY, globalWorkSizeZ/localWorkSizeZ, localWorkSizeX, localWorkSizeY, + f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, localWorkSizeX, localWorkSizeY, localWorkSizeZ, sharedMemBytes, hStream, kernelParams, extra, startEvent, stopEvent, flags)); } @@ -314,11 +312,8 @@ hipError_t hipHccModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX, HIP_INIT_API(hipHccModuleLaunchKernel, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, localWorkSizeX, localWorkSizeY, localWorkSizeZ, sharedMemBytes, hStream, kernelParams, extra); - if(localWorkSizeX == 0 || localWorkSizeY == 0 || localWorkSizeZ == 0) - return hipErrorInvalidValue; - return ihipLogStatus(ihipModuleLaunchKernel(tls, - f, globalWorkSizeX/localWorkSizeX, globalWorkSizeY/localWorkSizeY, globalWorkSizeZ/localWorkSizeZ, localWorkSizeX, localWorkSizeY, + f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, localWorkSizeX, localWorkSizeY, localWorkSizeZ, sharedMemBytes, hStream, kernelParams, extra, startEvent, stopEvent, 0)); } @@ -364,14 +359,26 @@ hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList } GET_TLS(); + + size_t globalWorkSizeX = 0, globalWorkSizeY = 0, globalWorkSizeZ = 0; + // launch kernels for each device for (int i = 0; i < numDevices; ++i) { const hipLaunchParams& lp = launchParamsList[i]; + globalWorkSizeX = (size_t)lp.gridDim.x * (size_t)lp.blockDim.x; + globalWorkSizeY = (size_t)lp.gridDim.y * (size_t)lp.blockDim.y; + globalWorkSizeZ = (size_t)lp.gridDim.z * (size_t)lp.blockDim.z; + + if(globalWorkSizeX > UINT32_MAX || globalWorkSizeY > UINT32_MAX || globalWorkSizeZ > UINT32_MAX) + { + return hipErrorInvalidConfiguration; + } + result = ihipModuleLaunchKernel(tls, kds[i], - lp.gridDim.x, - lp.gridDim.y, - lp.gridDim.z, + lp.gridDim.x * lp.blockDim.x, + lp.gridDim.y * lp.blockDim.y, + lp.gridDim.z * lp.blockDim.z, lp.blockDim.x, lp.blockDim.y, lp.blockDim.z, lp.sharedMem, lp.stream, lp.args, nullptr, nullptr, nullptr, 0, @@ -416,6 +423,14 @@ hipError_t ihipLaunchCooperativeKernel(const void* f, dim3 gridDim, return hipErrorInvalidConfiguration; } + size_t globalWorkSizeX = (size_t)gridDim.x * (size_t)blockDimX.x; + size_t globalWorkSizeY = (size_t)gridDim.y * (size_t)blockDimX.y; + size_t globalWorkSizeZ = (size_t)gridDim.z * (size_t)blockDimX.z; + if(globalWorkSizeX > UINT32_MAX || globalWorkSizeY > UINT32_MAX || globalWorkSizeZ > UINT32_MAX) + { + return hipErrorInvalidConfiguration; + } + // Prepare the kernel descriptor for initializing the GWS hipFunction_t gwsKD = ps.kernel_descriptor( reinterpret_cast(&init_gws), @@ -475,9 +490,9 @@ hipError_t ihipLaunchCooperativeKernel(const void* f, dim3 gridDim, // launch the main kernel result = ihipModuleLaunchKernel(tls, kd, - gridDim.x, - gridDim.y, - gridDim.z, + gridDim.x * blockDimX.x, + gridDim.y * blockDimX.y, + gridDim.z * blockDimX.z, blockDimX.x, blockDimX.y, blockDimX.z, sharedMemBytes, stream, kernelParams, nullptr, nullptr, nullptr, 0, true, impCoopParams); @@ -612,6 +627,8 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL void* impCoopParams[1]; ulong prev_sum = 0; + + size_t globalWorkSizeX = 0, globalWorkSizeY = 0, globalWorkSizeZ = 0; // launch the main kernels for each device for (int i = 0; i < numDevices; ++i) { const hipLaunchParams& lp = launchParamsList[i]; @@ -628,10 +645,18 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL impCoopParams[0] = &mg_info_ptr[i]; + globalWorkSizeX = (size_t)lp.gridDim.x * (size_t)lp.blockDim.x; + globalWorkSizeY = (size_t)lp.gridDim.y * (size_t)lp.blockDim.y; + globalWorkSizeZ = (size_t)lp.gridDim.z * (size_t)lp.blockDim.z; + if(globalWorkSizeX > UINT32_MAX || globalWorkSizeY > UINT32_MAX || globalWorkSizeZ > UINT32_MAX) + { + return hipErrorInvalidConfiguration; + } + result = ihipModuleLaunchKernel(tls, kds[i], - lp.gridDim.x, - lp.gridDim.y, - lp.gridDim.z, + lp.gridDim.x * lp.blockDim.x, + lp.gridDim.y * lp.blockDim.y, + lp.gridDim.z * lp.blockDim.z, lp.blockDim.x, lp.blockDim.y, lp.blockDim.z, lp.sharedMem, lp.stream, lp.args, nullptr, nullptr, nullptr, 0, From 03bb658721eddc233b9916293cc968d3f1d07a60 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Thu, 13 Feb 2020 00:52:56 -0800 Subject: [PATCH 024/154] missing break statement in hipDeviceGetAttribute (#1865) The break is missing for hipDeviceAttributeMaxTexture3DDepth. --- src/hip_device.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hip_device.cpp b/src/hip_device.cpp index aa89e62271..1bbdb10bbc 100644 --- a/src/hip_device.cpp +++ b/src/hip_device.cpp @@ -291,6 +291,7 @@ hipError_t ihipDeviceGetAttribute(int* pi, hipDeviceAttribute_t attr, int device break; case hipDeviceAttributeMaxTexture3DDepth: *pi = prop->maxTexture3D[2]; + break; case hipDeviceAttributeHdpMemFlushCntl: { uint32_t** hdp = reinterpret_cast(pi); From ead254cdd5d94cf4603c5a2ba41278de420c25da Mon Sep 17 00:00:00 2001 From: Satyanvesh Dittakavi Date: Thu, 13 Feb 2020 16:34:05 +0530 Subject: [PATCH 025/154] updated test for nvidia path --- .../module/hipModuleLoadDataMultThreaded.cpp | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 03a2b82b8b..6115125399 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -1,5 +1,5 @@ /* -Copyright (c) 2015-2016 Advanced Micro Devices, Inc. All rights reserved. +Copyright (c) 2015-Present Advanced Micro Devices, Inc. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc + * BUILD: %t %s ../../test_common.cpp * TEST: %t * HIT_END */ @@ -35,13 +35,11 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN << 2 -#define THREADS 8 +#define THREADS 2 #define FILENAME "vcpy_kernel.code" #define kernel_name "hello_world" -using ModuleFunction = std::pair; - std::vector load_file() { std::ifstream file(FILENAME, std::ios::binary | std::ios::ate); @@ -55,18 +53,18 @@ std::vector load_file() return buffer; } -ModuleFunction load(const std::vector& buffer) { +void run(const std::vector& buffer) { + hipDevice_t device; + HIPCHECK(hipDeviceGet(&device, 0)); + hipCtx_t context; + HIPCHECK(hipCtxCreate(&context, 0, device)); + hipModule_t Module; hipFunction_t Function; HIPCHECK(hipModuleLoadData(&Module, &buffer[0])); HIPCHECK(hipModuleGetFunction(&Function, Module, kernel_name)); - return {Module, Function}; -} - -void run(ModuleFunction mf) { - hipModule_t Module = mf.first; - hipFunction_t Function = mf.second; - float *A, *B, *Ad, *Bd; + + float *A, *B, *Ad, *Bd; A = new float[LEN]; B = new float[LEN]; @@ -105,6 +103,13 @@ void run(ModuleFunction mf) { for (uint32_t i = 0; i < LEN; i++) { assert(A[i] == B[i]); } + + hipFree(Ad); + hipFree(Bd); + delete A; + delete B; + hipCtxDestroy(context); + } struct joinable_thread : std::thread @@ -124,25 +129,23 @@ struct joinable_thread : std::thread } }; -void run_multi_threads(uint32_t n) { - std::vector mf(n); - { - auto buffer = load_file(); - std::vector threads; - for (uint32_t i = 0; i < n; i++) { - threads.emplace_back(std::thread{[&, i, buffer] { - mf[i] = load(buffer); - }}); - } +void run_multi_threads(uint32_t n, const std::vector& buffer) { + + std::vector threads; + + for (uint32_t i = 0; i < n; i++) { + threads.emplace_back(std::thread{[&, buffer] { + run(buffer); + }}); } - for(auto&& x:mf) - run(x); + } int main() { HIPCHECK(hipInit(0)); - run_multi_threads(THREADS * std::thread::hardware_concurrency()); + auto buffer = load_file(); + run_multi_threads(THREADS * std::thread::hardware_concurrency(), buffer); passed(); } From 3fb4135946572ecacfa1ab36c0629cd409a13081 Mon Sep 17 00:00:00 2001 From: Satyanvesh Dittakavi Date: Thu, 13 Feb 2020 19:48:26 +0530 Subject: [PATCH 026/154] Add c++11 option for nvcc --- tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 6115125399..8591a748df 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp + * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS -std=c++11 * TEST: %t * HIT_END */ From 084b2fa0f6de04f22832e924f17ea6cca6ebf16b Mon Sep 17 00:00:00 2001 From: Evgeny Mankov Date: Thu, 13 Feb 2020 18:34:10 +0300 Subject: [PATCH 027/154] [HIPIFY][doc] Update README.md: Windows tested configurations --- hipify-clang/README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/hipify-clang/README.md b/hipify-clang/README.md index 86a97dbe29..950482b7ed 100644 --- a/hipify-clang/README.md +++ b/hipify-clang/README.md @@ -393,19 +393,19 @@ Testing Time: 3.07s ``` ### hipify-clang: Windows -On Windows 10 the following configurations are tested: +*Tested configurations:* -LLVM 5.0.0 - 5.0.2, CUDA 8.0, cudnn 5.1.10 - 7.1.4.18 +| **LLVM** | **CUDA** | **cuDNN** | **Visual Studio** | **cmake** | **Python** | +|:--------------:|---------:|--------------------:|--------------------------:|----------:|-----------:| +| 5.0.0 - 5.0.2 | 8.0 | 5.1.10 - 7.1.4.18 | 2017.15.5.2 | 3.5.1 | 3.6.4 | +| 6.0.0 - 6.0.1 | 9.0 | 7.0.5.15 - 7.6.5.32 | 2017.15.5.5 | 3.6.0 | 3.7.2 | +| 7.0.0 - 7.1.0 | 9.2 | 7.6.5.32 | 2017.15.9.11 | 3.13.3 | 3.7.3 | +| 8.0.0 - 8.0.1 | 10.0 | 7.6.5.32 | 2017.15.9.15 | 3.14.2 | 3.7.4 | +| 9.0.0 - 9.0.1 | 10.1 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.0 | +| 10.0.0-rc1 | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.1 | +| 11.0.0git | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.1 | -LLVM 6.0.0 - 6.0.1, CUDA 9.0, cudnn 7.0.5.15 - 7.6.5.32 - -LLVM 7.0.0 - 10.0.0-rc1, CUDA 7.5 - 10.2, cudnn 7.0.5.15 - 7.6.5.32 - -Build system requirements for the latest stable configuration LLVM 9.0.1/CUDA 10.1 Update 2: - -Python 3.6.0 - 3.8.1, cmake 3.5.1 - 3.16.3, Visual Studio 2017 (15.5.2) - 2019 (16.4.4). - -Here is an example of building `hipify-clang` with testing support on `Windows 10` by `Visual Studio 16 2019`: +*Building with testing support on `Windows 10` by `Visual Studio 16 2019`:* ```shell cmake From 9d97f91fbb26a0102e467b290ca004a1fccfa482 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Thu, 13 Feb 2020 23:21:40 -0800 Subject: [PATCH 028/154] [sample] Add hipDispatchEnqueueRateMT (#1869) * [sample] Add hipDispatchEnqueueRateMT --- samples/1_Utils/hipDispatchLatency/Makefile | 11 +- .../hipDispatchEnqueueRateMT.cpp | 167 ++++++++++++++++++ 2 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 samples/1_Utils/hipDispatchLatency/hipDispatchEnqueueRateMT.cpp diff --git a/samples/1_Utils/hipDispatchLatency/Makefile b/samples/1_Utils/hipDispatchLatency/Makefile index 0616f01f0d..74945dc515 100644 --- a/samples/1_Utils/hipDispatchLatency/Makefile +++ b/samples/1_Utils/hipDispatchLatency/Makefile @@ -4,16 +4,17 @@ ifeq (,$(HIP_PATH)) endif HIPCC=$(HIP_PATH)/bin/hipcc -std=c++11 -EXE=hipDispatchLatency - CXXFLAGS = -O3 -all: test_kernel.code ${EXE} +all: test_kernel.code hipDispatchLatency.out hipDispatchEnqueueRateMT.out -$(EXE): hipDispatchLatency.cpp +hipDispatchLatency.out: hipDispatchLatency.cpp $(HIPCC) $(CXXFLAGS) hipDispatchLatency.cpp -o $@ +hipDispatchEnqueueRateMT.out: hipDispatchEnqueueRateMT.cpp + $(HIPCC) $(CXXFLAGS) hipDispatchEnqueueRateMT.cpp -o $@ + test_kernel.code: test_kernel.cpp $(HIP_PATH)/bin/hipcc --genco $(GENCO_FLAGS) $^ -o $@ clean: - rm -f *.o $(EXE) + rm -f *.o *.out diff --git a/samples/1_Utils/hipDispatchLatency/hipDispatchEnqueueRateMT.cpp b/samples/1_Utils/hipDispatchLatency/hipDispatchEnqueueRateMT.cpp new file mode 100644 index 0000000000..d1b5c2f3b5 --- /dev/null +++ b/samples/1_Utils/hipDispatchLatency/hipDispatchEnqueueRateMT.cpp @@ -0,0 +1,167 @@ +/* +Copyright (c) 2020-present Advanced Micro Devices, Inc. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include "hip/hip_runtime.h" +#ifdef __HIP_PLATFORM_HCC__ +#include "hip/hip_ext.h" +#endif +#include +#include +#include +#include +#include +#include +#include + +#define NUM_GROUPS 1 +#define GROUP_SIZE 1 +#define WARMUP_RUN_COUNT 10 +#define TIMING_RUN_COUNT 100 +#define TOTAL_RUN_COUNT WARMUP_RUN_COUNT + TIMING_RUN_COUNT + +__global__ void EmptyKernel() {} + +// Helper to print various timing metrics +void print_timing(std::string test, std::array &results, int batch = 1) +{ + + float total_us = 0.0f, mean_us = 0.0f, stddev_us = 0.0f; + + // remove top outliers due to nature of variability across large number of multi-threaded runs + std::sort(results.begin(), results.end(), std::greater()); + auto start_iter = std::next(results.begin(), WARMUP_RUN_COUNT); + auto end_iter = results.end(); + + // mean + std::for_each(start_iter, end_iter, [&](const float &run_ms) { + total_us += (run_ms * 1000) / batch; + }); + mean_us = total_us / TIMING_RUN_COUNT; + + // stddev + total_us = 0; + std::for_each(start_iter, end_iter, [&](const float &run_ms) { + float dev_us = ((run_ms * 1000) / batch) - mean_us; + total_us += dev_us * dev_us; + }); + stddev_us = sqrt(total_us / TIMING_RUN_COUNT); + + printf("\n %s: %.1f us, std: %.1f us\n", test.c_str(), mean_us, stddev_us); +} + +// Measure time taken to enqueue a kernel on the GPU using hipModuleLaunchKernel +void hipModuleLaunchKernel_enqueue_rate(std::atomic_int* shared, int max_threads) +{ + //resources necessary for this thread + hipStream_t stream; + hipStreamCreate(&stream); + hipModule_t module; + hipFunction_t function; + hipModuleLoad(&module, "test_kernel.code"); + hipModuleGetFunction(&function, module, "test"); + void* kernel_params = nullptr; + std::array results; + + //synchronize all threads, before running + int tid = shared->fetch_add(1, std::memory_order_release); + while (max_threads != shared->load(std::memory_order_acquire)) {} + + for (auto i = 0; i < TOTAL_RUN_COUNT; ++i) { + auto start = std::chrono::high_resolution_clock::now(); + hipModuleLaunchKernel(function, 1, 1, 1, 1, 1, 1, 0, stream, &kernel_params, nullptr); + auto stop = std::chrono::high_resolution_clock::now(); + results[i] = std::chrono::duration(stop - start).count(); + } + print_timing("Thread ID : " + std::to_string(tid) + " , " + "hipModuleLaunchKernel enqueue rate", results); +} + +// Measure time taken to enqueue a kernel on the GPU using hipLaunchKernelGGL +void hipLaunchKernelGGL_enqueue_rate(std::atomic_int* shared, int max_threads) +{ + //resources necessary for this thread + hipStream_t stream; + hipStreamCreate(&stream); + std::array results; + + //synchronize all threads, before running + int tid = shared->fetch_add(1, std::memory_order_release); + while (max_threads != shared->load(std::memory_order_acquire)) {} + + for (auto i = 0; i < TOTAL_RUN_COUNT; ++i) { + auto start = std::chrono::high_resolution_clock::now(); + hipLaunchKernelGGL((EmptyKernel), dim3(NUM_GROUPS), dim3(GROUP_SIZE), 0, stream); + auto stop = std::chrono::high_resolution_clock::now(); + results[i] = std::chrono::duration(stop - start).count(); + } + print_timing("Thread ID : " + std::to_string(tid) + " , " + "hipLaunchKernelGGL enqueue rate", results); +} + +// Simple thread pool +struct thread_pool { + thread_pool(int total_threads) : max_threads(total_threads) {} + void start(std::function f) { + for (int i = 0; i < max_threads; ++i) { + threads.push_back(std::async(std::launch::async, f, &shared, max_threads)); + } + } + void finish() { + for (auto&&thread : threads) { + thread.get(); + } + threads.clear(); + shared = {0}; + } + ~thread_pool() { + finish(); + } +private: + std::atomic_int shared {0}; + std::vector> threads; + int max_threads = 1; +}; + + +int main(int argc, char* argv[]) +{ + if (argc != 3) { + std::cerr << "Run test as 'hipDispatchEnqueueRateMT <0-hipModuleLaunchKernel /1-hipLaunchKernelGGL>'\n"; + return -1; + } + + int max_threads = atoi(argv[1]); + int run_module_test = atoi(argv[2]); + if(max_threads < 1 || run_module_test < 0 || run_module_test > 1) { + std::cerr << "Invalid Input.\n"; + std::cerr << "Run test as 'hipDispatchEnqueueRateMT <0-hipModuleLaunchKernel /1-hipLaunchKernelGGL>'\n"; + return -1; + } + thread_pool task(max_threads); + + if(run_module_test == 0) { + task.start(hipModuleLaunchKernel_enqueue_rate); + task.finish(); + } else { + task.start(hipLaunchKernelGGL_enqueue_rate); + task.finish(); + } + + return 0; +} + From 115f45d116d8c732aa5da56e08ef98f719b393f7 Mon Sep 17 00:00:00 2001 From: Evgeny Mankov Date: Fri, 14 Feb 2020 13:09:31 +0300 Subject: [PATCH 029/154] [HIPIFY][doc] Update README.md: LLVM 10.0.0-rc2 - the latest supported LLVM Release --- hipify-clang/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hipify-clang/README.md b/hipify-clang/README.md index 950482b7ed..a375c6ab65 100644 --- a/hipify-clang/README.md +++ b/hipify-clang/README.md @@ -42,10 +42,10 @@ After applying all the matchers, the output HIP source is produced. `hipify-clang` requires: -1. [**LLVM+CLANG**](http://releases.llvm.org) of at least version [3.8.0](http://releases.llvm.org/download.html#3.8.0); the latest stable and recommended release: [**9.0.1**](http://releases.llvm.org/download.html#9.0.1), the latest release candidate: [10.0.0-rc1](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc1). +1. [**LLVM+CLANG**](http://releases.llvm.org) of at least version [3.8.0](http://releases.llvm.org/download.html#3.8.0); the latest stable and recommended release: [**9.0.1**](http://releases.llvm.org/download.html#9.0.1), the latest release candidate: [10.0.0-rc2](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc2). 2. [**CUDA**](https://developer.nvidia.com/cuda-downloads) of at least version [7.0](https://developer.nvidia.com/cuda-toolkit-70), the latest supported version is [**10.1 Update 2**](https://developer.nvidia.com/cuda-10.1-download-archive-base). -To use the latest CUDA version [10.2](https://developer.nvidia.com/cuda-downloads) please use the latest `LLVM` release candidate: [10.0.0-rc1](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc1). +To use the latest CUDA version [10.2](https://developer.nvidia.com/cuda-downloads) please use the latest `LLVM` release candidate: [10.0.0-rc2](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc2). | **LLVM release version** | **CUDA latest supported version** | **Windows** | **Linux** | |:----------------------------------------------------------:|:------------------------------------------------------------------------:|:-----------:|:---------:| @@ -67,7 +67,7 @@ To use the latest CUDA version [10.2](https://developer.nvidia.com/cuda-download | [8.0.1](http://releases.llvm.org/download.html#8.0.1) | [10.0](https://developer.nvidia.com/cuda-10.0-download-archive) | -
not working due to
the clang's bug [38811](https://bugs.llvm.org/show_bug.cgi?id=38811)
+
[patch](patches/patch_for_clang_8.0.1_bug_38811.zip)*
| + | | [9.0.0](http://releases.llvm.org/download.html#9.0.0) | [10.1](https://developer.nvidia.com/cuda-10.1-download-archive-base) | + | + | | [**9.0.1**](http://releases.llvm.org/download.html#9.0.1) | [**10.1**](https://developer.nvidia.com/cuda-10.1-download-archive-base) | +
**LATEST STABLE RELEASE** | +
**LATEST STABLE RELEASE** | -| [10.0.0-rc1](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc1) | [10.2](https://developer.nvidia.com/cuda-downloads) | + | + | +| [10.0.0-rc2](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc2) | [10.2](https://developer.nvidia.com/cuda-downloads) | + | + | `*` Download the patch and unpack it into your `LLVM` distributive directory; a few header files will be overwritten; rebuilding of `LLVM` is not needed. @@ -158,7 +158,7 @@ Run `Visual Studio 16 2019`, open the generated `LLVM.sln`, build all, build pro **LLVM 10.0.0 or newer:** -1. download [`LLVM project`](https://github.com/llvm/llvm-project/archive/llvmorg-10.0.0-rc1.tar.gz) sources; +1. download [`LLVM project`](https://github.com/llvm/llvm-project/archive/llvmorg-10.0.0-rc2.tar.gz) sources; 2. build [`LLVM project`](http://llvm.org/docs/CMake.html): **Linux**: @@ -247,7 +247,7 @@ On Linux the following configurations are tested: Ubuntu 14: LLVM 5.0.0 - 6.0.1, CUDA 7.0 - 9.0, cudnn-5.0.5 - cudnn-7.6.5.32 -Ubuntu 16-18: LLVM 8.0.0 - 10.0.0-rc1, CUDA 8.0 - 10.2, cudnn-5.1.10 - cudnn-7.6.5.32 +Ubuntu 16-18: LLVM 8.0.0 - 10.0.0-rc2, CUDA 8.0 - 10.2, cudnn-5.1.10 - cudnn-7.6.5.32 Minimum build system requirements for the above configurations: @@ -402,7 +402,7 @@ Testing Time: 3.07s | 7.0.0 - 7.1.0 | 9.2 | 7.6.5.32 | 2017.15.9.11 | 3.13.3 | 3.7.3 | | 8.0.0 - 8.0.1 | 10.0 | 7.6.5.32 | 2017.15.9.15 | 3.14.2 | 3.7.4 | | 9.0.0 - 9.0.1 | 10.1 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.0 | -| 10.0.0-rc1 | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.1 | +| 10.0.0-rc1,rc2 | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.1 | | 11.0.0git | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.1 | *Building with testing support on `Windows 10` by `Visual Studio 16 2019`:* From f2ab87d872b88b4abd1c3c8871052321042b72a9 Mon Sep 17 00:00:00 2001 From: Siu Chi Chan Date: Fri, 14 Feb 2020 13:20:12 -0500 Subject: [PATCH 030/154] Disabling HCC code object v3 generation by default. Some PyTorch unit tests have regression. Disabling cov3 to allow more time to debug and unblock PyTorch Change-Id: Iba7f425ef3499c20c42ec45d9152b5d27ce97d03 --- bin/hipcc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/hipcc b/bin/hipcc index eb3f185424..1bdb466c68 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -342,7 +342,7 @@ my $runCmd = 1; my $buildDeps = 0; my $linkType = 1; my $setLinkType = 0; -my $coFormatv3 = 1; +my $coFormatv3 = 0; my @options = (); my @inputs = (); From 797a929a652f15357a4bda26b88eb81044484a94 Mon Sep 17 00:00:00 2001 From: Nick Curtis Date: Fri, 14 Feb 2020 22:21:09 -0600 Subject: [PATCH 031/154] Implement long / long long shuffles (#1829) Implement additional data-types for shuffles (long and long long). Based upon the double implementation. --- include/hip/hcc_detail/device_functions.h | 114 +++++++++++++++++++ tests/src/kernel/hipShflTests.cpp | 130 ++++++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 tests/src/kernel/hipShflTests.cpp diff --git a/include/hip/hcc_detail/device_functions.h b/include/hip/hcc_detail/device_functions.h index 68e3277270..46ed53ff87 100644 --- a/include/hip/hcc_detail/device_functions.h +++ b/include/hip/hcc_detail/device_functions.h @@ -319,6 +319,36 @@ double __shfl(double var, int src_lane, int width = warpSize) { double tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); return tmp1; } +__device__ +inline +long __shfl(long var, int src_lane, int width = warpSize) +{ + static_assert(sizeof(long) == 2 * sizeof(int), ""); + static_assert(sizeof(long) == sizeof(uint64_t), ""); + + int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); + tmp[0] = __shfl(tmp[0], src_lane, width); + tmp[1] = __shfl(tmp[1], src_lane, width); + + uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); + long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); + return tmp1; +} +__device__ +inline +long long __shfl(long long var, int src_lane, int width = warpSize) +{ + static_assert(sizeof(long long) == 2 * sizeof(int), ""); + static_assert(sizeof(long long) == sizeof(uint64_t), ""); + + int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); + tmp[0] = __shfl(tmp[0], src_lane, width); + tmp[1] = __shfl(tmp[1], src_lane, width); + + uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); + long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); + return tmp1; +} __device__ inline @@ -356,6 +386,34 @@ double __shfl_up(double var, unsigned int lane_delta, int width = warpSize) { double tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); return tmp1; } +__device__ +inline +long __shfl_up(long var, unsigned int lane_delta, int width = warpSize) +{ + static_assert(sizeof(long) == 2 * sizeof(int), ""); + static_assert(sizeof(long) == sizeof(uint64_t), ""); + + int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); + tmp[0] = __shfl_up(tmp[0], lane_delta, width); + tmp[1] = __shfl_up(tmp[1], lane_delta, width); + + uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); + long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); + return tmp1; +} +__device__ +inline +long long __shfl_up(long long var, unsigned int lane_delta, int width = warpSize) +{ + static_assert(sizeof(long long) == 2 * sizeof(int), ""); + static_assert(sizeof(long long) == sizeof(uint64_t), ""); + int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); + tmp[0] = __shfl_up(tmp[0], lane_delta, width); + tmp[1] = __shfl_up(tmp[1], lane_delta, width); + uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); + long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); + return tmp1; +} __device__ inline @@ -393,6 +451,34 @@ double __shfl_down(double var, unsigned int lane_delta, int width = warpSize) { double tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); return tmp1; } +__device__ +inline +long __shfl_down(long var, unsigned int lane_delta, int width = warpSize) +{ + static_assert(sizeof(long) == 2 * sizeof(int), ""); + static_assert(sizeof(long) == sizeof(uint64_t), ""); + + int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); + tmp[0] = __shfl_down(tmp[0], lane_delta, width); + tmp[1] = __shfl_down(tmp[1], lane_delta, width); + + uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); + long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); + return tmp1; +} +__device__ +inline +long long __shfl_down(long long var, unsigned int lane_delta, int width = warpSize) +{ + static_assert(sizeof(long long) == 2 * sizeof(int), ""); + static_assert(sizeof(long long) == sizeof(uint64_t), ""); + int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); + tmp[0] = __shfl_down(tmp[0], lane_delta, width); + tmp[1] = __shfl_down(tmp[1], lane_delta, width); + uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); + long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); + return tmp1; +} __device__ inline @@ -430,6 +516,34 @@ double __shfl_xor(double var, int lane_mask, int width = warpSize) { double tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); return tmp1; } +__device__ +inline +long __shfl_xor(long var, int lane_mask, int width = warpSize) +{ + static_assert(sizeof(long) == 2 * sizeof(int), ""); + static_assert(sizeof(long) == sizeof(uint64_t), ""); + + int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); + tmp[0] = __shfl_xor(tmp[0], lane_mask, width); + tmp[1] = __shfl_xor(tmp[1], lane_mask, width); + + uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); + long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); + return tmp1; +} +__device__ +inline +long long __shfl_xor(long long var, int lane_mask, int width = warpSize) +{ + static_assert(sizeof(long long) == 2 * sizeof(int), ""); + static_assert(sizeof(long long) == sizeof(uint64_t), ""); + int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp)); + tmp[0] = __shfl_xor(tmp[0], lane_mask, width); + tmp[1] = __shfl_xor(tmp[1], lane_mask, width); + uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); + long long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); + return tmp1; +} #define MASK1 0x00ff00ff #define MASK2 0xff00ff00 diff --git a/tests/src/kernel/hipShflTests.cpp b/tests/src/kernel/hipShflTests.cpp new file mode 100644 index 0000000000..9b1cc73248 --- /dev/null +++ b/tests/src/kernel/hipShflTests.cpp @@ -0,0 +1,130 @@ +/* +Copyright (c) 2015-present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +/* HIT_START + * BUILD: %t %s ../test_common.cpp + * TEST: %t + * HIT_END + */ + +#include +#include +#include "test_common.h" + +#define WIDTH 4 + +#define NUM (WIDTH * WIDTH) + +#define THREADS_PER_BLOCK_X 4 +#define THREADS_PER_BLOCK_Y 4 +#define THREADS_PER_BLOCK_Z 1 + +// Device (Kernel) function, it must be void +template +__global__ void matrixTranspose(T* out, T* in, const int width) { + int x = hipBlockDim_x * hipBlockIdx_x + hipThreadIdx_x; + T val = in[x]; + for (int i = 0; i < width; i++) { + for (int j = 0; j < width; j++) out[i * width + j] = __shfl(val, j * width + i); + } +} + +// CPU implementation of matrix transpose +template +void matrixTransposeCPUReference(T* output, T* input, const unsigned int width) { + for (unsigned int j = 0; j < width; j++) { + for (unsigned int i = 0; i < width; i++) { + output[i * width + j] = input[j * width + i]; + } + } +} + +template +void runTest() { + T* Matrix; + T* TransposeMatrix; + T* cpuTransposeMatrix; + + T* gpuMatrix; + T* gpuTransposeMatrix; + + hipDeviceProp_t devProp; + hipGetDeviceProperties(&devProp, 0); + + int i; + int errors; + + Matrix = (T*)malloc(NUM * sizeof(T)); + TransposeMatrix = (T*)malloc(NUM * sizeof(T)); + cpuTransposeMatrix = (T*)malloc(NUM * sizeof(T)); + + // initialize the input data + for (i = 0; i < NUM; i++) { + Matrix[i] = (T)i * 10l; + } + + // allocate the memory on the device side + hipMalloc((void**)&gpuMatrix, NUM * sizeof(T)); + hipMalloc((void**)&gpuTransposeMatrix, NUM * sizeof(T)); + + // Memory transfer from host to device + hipMemcpy(gpuMatrix, Matrix, NUM * sizeof(T), hipMemcpyHostToDevice); + + // Lauching kernel from host + hipLaunchKernelGGL(matrixTranspose, dim3(1), dim3(THREADS_PER_BLOCK_X * THREADS_PER_BLOCK_Y), 0, 0, + gpuTransposeMatrix, gpuMatrix, WIDTH); + + // Memory transfer from device to host + hipMemcpy(TransposeMatrix, gpuTransposeMatrix, NUM * sizeof(T), hipMemcpyDeviceToHost); + + // CPU MatrixTranspose computation + matrixTransposeCPUReference(cpuTransposeMatrix, Matrix, WIDTH); + + // verify the results + errors = 0; + double eps = 1.0E-6; + for (i = 0; i < NUM; i++) { + if (TransposeMatrix[i] != cpuTransposeMatrix[i]) { + errors++; + } + } + + // free the resources on device side + hipFree(gpuMatrix); + hipFree(gpuTransposeMatrix); + + // free the resources on host side + free(Matrix); + free(TransposeMatrix); + free(cpuTransposeMatrix); + + if (errors != 0) { + failed("Mismatch present"); + } +} + +int main() { + runTest(); + runTest(); + runTest(); + runTest(); + passed(); +} From 56b8b0d80ea87097355f7eee04f2410b2c96e557 Mon Sep 17 00:00:00 2001 From: vsytch Date: Fri, 14 Feb 2020 23:22:25 -0500 Subject: [PATCH 032/154] Add missing __hip_pinned_shadow__ attributes to the texture global vars. (#1866) --- tests/src/texture/hipBindTex2DPitch.cpp | 4 ++++ tests/src/texture/hipNormalizedFloatValueTex.cpp | 3 +++ tests/src/texture/simpleTexture2DLayered.cpp | 6 ++++++ tests/src/texture/simpleTexture3D.cpp | 11 +++++++++++ 4 files changed, 24 insertions(+) diff --git a/tests/src/texture/hipBindTex2DPitch.cpp b/tests/src/texture/hipBindTex2DPitch.cpp index 905e24bce3..b01402c91d 100644 --- a/tests/src/texture/hipBindTex2DPitch.cpp +++ b/tests/src/texture/hipBindTex2DPitch.cpp @@ -27,6 +27,10 @@ THE SOFTWARE. #define SIZE_H 8 #define SIZE_W 12 #define TYPE_t float + +#if __HIP__ +__hip_pinned_shadow__ +#endif texture tex; // texture object is a kernel argument diff --git a/tests/src/texture/hipNormalizedFloatValueTex.cpp b/tests/src/texture/hipNormalizedFloatValueTex.cpp index 3179f7412e..609f6916f8 100644 --- a/tests/src/texture/hipNormalizedFloatValueTex.cpp +++ b/tests/src/texture/hipNormalizedFloatValueTex.cpp @@ -45,6 +45,9 @@ static float getNormalizedValue(const float value, } } +#if __HIP__ +__hip_pinned_shadow__ +#endif texture textureNormalizedVal_1D; __global__ void normalizedValTextureTest(unsigned int numElements, float* pDst) diff --git a/tests/src/texture/simpleTexture2DLayered.cpp b/tests/src/texture/simpleTexture2DLayered.cpp index e3ba6d9afe..fa545cb4ca 100644 --- a/tests/src/texture/simpleTexture2DLayered.cpp +++ b/tests/src/texture/simpleTexture2DLayered.cpp @@ -26,9 +26,15 @@ THE SOFTWARE. * HIT_END */ #include "test_common.h" + typedef float T; + // Texture reference for 2D Layered texture +#if __HIP__ +__hip_pinned_shadow__ +#endif texture tex2DL; + __global__ void simpleKernelLayeredArray(T* outputData,int width,int height,int layer) { unsigned int x = blockIdx.x*blockDim.x + threadIdx.x; diff --git a/tests/src/texture/simpleTexture3D.cpp b/tests/src/texture/simpleTexture3D.cpp index 741b1671af..06833761c8 100644 --- a/tests/src/texture/simpleTexture3D.cpp +++ b/tests/src/texture/simpleTexture3D.cpp @@ -31,8 +31,19 @@ THE SOFTWARE. const char *sampleName = "simpleTexture3D"; // Texture reference for 3D texture +#if __HIP__ +__hip_pinned_shadow__ +#endif texture texf; + +#if __HIP__ +__hip_pinned_shadow__ +#endif texture texi; + +#if __HIP__ +__hip_pinned_shadow__ +#endif texture texc; template From 854afef281096d0f01c091cde5a83d47aaecf960 Mon Sep 17 00:00:00 2001 From: Maneesh Gupta Date: Mon, 17 Feb 2020 11:16:20 +0530 Subject: [PATCH 033/154] [dtests] Fix random timeout failures in hipModuleLoadDataMultThreaded (#1877) Limit the max threads that are launched to 16. --- tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index 8591a748df..e73bbedba5 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -36,6 +36,7 @@ THE SOFTWARE. #define LEN 64 #define SIZE LEN << 2 #define THREADS 2 +#define MAX_THREADS 16 #define FILENAME "vcpy_kernel.code" #define kernel_name "hello_world" @@ -145,7 +146,7 @@ int main() { HIPCHECK(hipInit(0)); auto buffer = load_file(); - run_multi_threads(THREADS * std::thread::hardware_concurrency(), buffer); + run_multi_threads(min(THREADS * std::thread::hardware_concurrency(), MAX_THREADS), buffer); passed(); } From 8c5e5e435b5c5109c378eac083951f87b60899ed Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Mon, 17 Feb 2020 06:05:35 -0800 Subject: [PATCH 034/154] Fix hipMemcpy3D (#1798) Fixes #1790 and #1791. hipMemcpy3D still requires further refactoring for different input and output combinations. --- include/hip/hcc_detail/driver_types.h | 9 +- src/hip_memory.cpp | 205 +++++++++++-------- tests/src/runtimeApi/memory/hipMemcpy3D.cpp | 110 ++++++++++ tests/src/texture/simpleTexture2DLayered.cpp | 2 +- tests/src/texture/simpleTexture3D.cpp | 39 ++-- 5 files changed, 249 insertions(+), 116 deletions(-) create mode 100644 tests/src/runtimeApi/memory/hipMemcpy3D.cpp diff --git a/include/hip/hcc_detail/driver_types.h b/include/hip/hcc_detail/driver_types.h index 0c29542c7e..1941f44617 100644 --- a/include/hip/hcc_detail/driver_types.h +++ b/include/hip/hcc_detail/driver_types.h @@ -255,14 +255,14 @@ typedef struct hipMemcpy3DParms { hipArray_t srcArray; struct hipPos srcPos; struct hipPitchedPtr srcPtr; - hipArray_t dstArray; struct hipPos dstPos; struct hipPitchedPtr dstPtr; - struct hipExtent extent; enum hipMemcpyKind kind; +} hipMemcpy3DParms; +typedef struct HIP_MEMCPY3D { size_t Depth; size_t Height; size_t WidthInBytes; @@ -283,10 +283,7 @@ typedef struct hipMemcpy3DParms { size_t srcLOD; hipMemoryType srcMemoryType; size_t srcPitch; - size_t srcXInBytes; - size_t srcY; - size_t srcZ; -}hipMemcpy3DParms; +} HIP_MEMCPY3D; static inline struct hipPitchedPtr make_hipPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) { diff --git a/src/hip_memory.cpp b/src/hip_memory.cpp index 444e41107a..d965059923 100644 --- a/src/hip_memory.cpp +++ b/src/hip_memory.cpp @@ -1540,111 +1540,144 @@ hipError_t hipMemcpyAtoH(void* dst, hipArray* srcArray, size_t srcOffset, size_t return ihipLogStatus(e); } +int getByteSizeFromFormat(const hipChannelFormatDesc& desc){ + int byteSize =0; + switch (desc.f) { + case hipChannelFormatKindUnsigned: + switch (desc.x) { + case 32: + byteSize = sizeof(uint32_t); + break; + case 16: + byteSize = sizeof(uint16_t); + break; + case 8: + byteSize = sizeof(uint8_t); + break; + default: + byteSize = sizeof(uint32_t); + } + break; + case hipChannelFormatKindSigned: + switch (desc.x) { + case 32: + byteSize = sizeof(int32_t); + break; + case 16: + byteSize = sizeof(int16_t); + break; + case 8: + byteSize = sizeof(int8_t); + break; + default: + byteSize = sizeof(int32_t); + } + break; + case hipChannelFormatKindFloat: + switch (desc.x) { + case 32: + byteSize = sizeof(float); + break; + case 16: + byteSize = sizeof(_Float16); + break; + default: + byteSize = sizeof(float); + } + break; + case hipChannelFormatKindNone: + default: + break; + } + return byteSize; +} + hipError_t ihipMemcpy3D(const struct hipMemcpy3DParms* p, hipStream_t stream, bool isAsync) { hipError_t e = hipSuccess; if(p) { - size_t byteSize, width, height, depth, widthInBytes, srcPitch, dstPitch, ySize; - hipChannelFormatDesc desc; - void* srcPtr;void* dstPtr; + size_t dstByteSize, srcByteSize, copyWidth, copyHeight, copyDepth, widthInBytes, srcPitch, dstPitch, srcYsize, dstYsize; + size_t srcXoffset, srcYoffset, srcZoffset, dstXoffset, dstYoffset, dstZoffset; + size_t srcWidth, srcHeight, srcDepth, dstWidth, dstHeight, dstDepth; + + void* srcPtr, *dstPtr; + bool copyWidthUpdate= false; + copyDepth = p->extent.depth; + copyHeight = p->extent.height; + copyWidth = p->extent.width; // in bytes ? + dstXoffset = p->dstPos.x; + dstYoffset = p->dstPos.y; + dstZoffset = p->dstPos.z; + srcXoffset = p->srcPos.x; + srcYoffset = p->srcPos.y; + srcZoffset = p->srcPos.z; if (p->dstArray != nullptr) { - if (p->dstArray->isDrv == false) { - switch (p->dstArray->desc.f) { - case hipChannelFormatKindSigned: - byteSize = sizeof(int); - break; - case hipChannelFormatKindUnsigned: - byteSize = sizeof(unsigned int); - break; - case hipChannelFormatKindFloat: - byteSize = sizeof(float); - break; - case hipChannelFormatKindNone: - byteSize = sizeof(size_t); - break; - default: - byteSize = 0; - break; - } - depth = p->extent.depth; - height = p->extent.height; - width = p->extent.width; - widthInBytes = p->extent.width * byteSize; - srcPitch = p->srcPtr.pitch; - srcPtr = p->srcPtr.ptr; - ySize = p->srcPtr.ysize; - desc = p->dstArray->desc; - dstPtr = p->dstArray->data; - hsa_ext_image_data_info_t imageInfo; - if(hipTextureType2DLayered == p->dstArray->textureType) - GetImageInfo(HSA_EXT_IMAGE_GEOMETRY_2DA, width, height, 0, desc, imageInfo, depth); - else - GetImageInfo(HSA_EXT_IMAGE_GEOMETRY_3D, width, height, depth, desc, imageInfo); - dstPitch = imageInfo.size/(height == 0 ? 1 : height)/(depth == 0 ? 1 : depth); - } else { - depth = p->Depth; - height = p->Height; - widthInBytes = p->WidthInBytes; - width = p->dstArray->width; - hsa_ext_image_channel_order_t channelOrder; - switch(p->dstArray->NumChannels) { - case 2: - channelOrder = HSA_EXT_IMAGE_CHANNEL_ORDER_RG; - break; - case 3: - channelOrder = HSA_EXT_IMAGE_CHANNEL_ORDER_RGB; - break; - case 4: - channelOrder = HSA_EXT_IMAGE_CHANNEL_ORDER_RGBA; - break; - case 1: - default: - channelOrder = HSA_EXT_IMAGE_CHANNEL_ORDER_R; - break; - } - hsa_ext_image_channel_type_t channelType; - e = ihipArrayToImageFormat(p->dstArray->Format,channelType); - srcPitch = p->srcPitch; - srcPtr = (void*)p->srcHost; - ySize = p->srcHeight; - dstPtr = p->dstArray->data; - hsa_ext_image_data_info_t imageInfo; - if(hipTextureType2DLayered == p->dstArray->textureType) - GetImageInfo(HSA_EXT_IMAGE_GEOMETRY_2DA, width, height, 0, channelOrder, channelType, imageInfo, depth); - else - GetImageInfo(HSA_EXT_IMAGE_GEOMETRY_3D, width, height, depth, channelOrder, channelType, imageInfo); - dstPitch = imageInfo.size/(height == 0 ? 1 : height)/(depth == 0 ? 1 : depth); + if ((p->dstArray->isDrv == true) ||( p->dstPtr.ptr!= nullptr)){ + return hipErrorInvalidValue; + } + // Array destination + dstByteSize = getByteSizeFromFormat(p->dstArray->desc); + hipChannelFormatDesc desc; + desc = p->dstArray->desc; + dstPtr = p->dstArray->data; + dstWidth = p->dstArray->width; + dstHeight = p->dstArray->height; + dstDepth = p->dstArray->depth; + dstPitch = dstByteSize * alignUp(dstWidth, IMAGE_PITCH_ALIGNMENT); + if(!copyWidthUpdate) { + copyWidth = copyWidth * dstByteSize; + copyWidthUpdate = true; } } else { - // Non array destination - depth = p->extent.depth; - height = p->extent.height; - widthInBytes = p->extent.width; - srcPitch = p->srcPtr.pitch; - srcPtr = p->srcPtr.ptr; + //Non Array destination dstPtr = p->dstPtr.ptr; - ySize = p->srcPtr.ysize; + dstWidth = p->dstPtr.xsize; + dstHeight = p->dstPtr.ysize; dstPitch = p->dstPtr.pitch; } + if (p->srcArray != nullptr) { + if ((p->srcArray->isDrv == true) ||( p->srcPtr.ptr!= nullptr)){ + return hipErrorInvalidValue; + } + // Array source + srcByteSize = getByteSizeFromFormat(p->srcArray->desc); + hipChannelFormatDesc desc; + desc = p->srcArray->desc; + srcPtr = p->srcArray->data; + srcWidth = p->srcArray->width; + srcHeight = p->srcArray->height; + srcDepth = p->srcArray->depth; + srcPitch = srcByteSize * alignUp(srcWidth, IMAGE_PITCH_ALIGNMENT); + if(!copyWidthUpdate) { + copyWidth = copyWidth * srcByteSize; + copyWidthUpdate = true; + } + } else { + //Non Array source + srcPtr = p->srcPtr.ptr; + srcWidth = p->srcPtr.xsize; + srcHeight = p->srcPtr.ysize; + srcPitch = p->srcPtr.pitch; + } + stream = ihipSyncAndResolveStream(stream); try { - if((widthInBytes == dstPitch) && (widthInBytes == srcPitch)) { + if((copyWidth == dstPitch) && (copyWidth == srcPitch)&& (copyHeight == dstHeight) &&(copyHeight == srcHeight)) { if(isAsync) - stream->locked_copyAsync((void*)dstPtr, (void*)srcPtr, widthInBytes*height*depth, p->kind); + stream->locked_copyAsync((void*)dstPtr, (void*)srcPtr, copyWidth*copyHeight*copyDepth, p->kind); else - stream->locked_copySync((void*)dstPtr, (void*)srcPtr, widthInBytes*height*depth, p->kind, false); + stream->locked_copySync((void*)dstPtr, (void*)srcPtr, copyWidth*copyHeight*copyDepth, p->kind, false); } else { - for (int i = 0; i < depth; i++) { - for (int j = 0; j < height; j++) { - // TODO: p->srcPos or p->dstPos are not 0. + for (int i = 0; i < copyDepth; i++) { + for (int j = 0; j < copyHeight; j++) { unsigned char* src = - (unsigned char*)srcPtr + i * ySize * srcPitch + j * srcPitch; + (unsigned char*)srcPtr + (i + srcZoffset) * srcHeight * srcPitch + (j + srcYoffset) * srcPitch + srcXoffset; unsigned char* dst = - (unsigned char*)dstPtr + i * height * dstPitch + j * dstPitch; + (unsigned char*)dstPtr + (i + dstZoffset) * dstHeight * dstPitch + (j + dstYoffset) * dstPitch + dstXoffset; if(isAsync) - stream->locked_copyAsync(dst, src, widthInBytes, p->kind); + stream->locked_copyAsync(dst, src, copyWidth, p->kind); else - stream->locked_copySync(dst, src, widthInBytes, p->kind); + stream->locked_copySync(dst, src, copyWidth, p->kind); } } } diff --git a/tests/src/runtimeApi/memory/hipMemcpy3D.cpp b/tests/src/runtimeApi/memory/hipMemcpy3D.cpp new file mode 100644 index 0000000000..255a3490b6 --- /dev/null +++ b/tests/src/runtimeApi/memory/hipMemcpy3D.cpp @@ -0,0 +1,110 @@ +/* +Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/* HIT_START + * BUILD: %t %s ../../test_common.cpp + * TEST: %t + * HIT_END + */ + +#include "test_common.h" + +template +void runTest(int width,int height,int depth, hipChannelFormatKind formatKind) +{ + unsigned int size = width * height * depth * sizeof(T); + T* hData = (T*) malloc(size); + memset(hData, 0, size); + + for (int i = 0; i < depth; i++) { + for (int j = 0; j < height; j++) { + for (int k = 0; k < width; k++) { + hData[i*width*height + j*width +k] = i*width*height + j*width + k; + } + } + } + printf("test- sizeof(T) =%d\n", sizeof(T)); + hipChannelFormatDesc channelDesc = hipCreateChannelDesc(sizeof(T)*8, 0, 0, 0, formatKind); + hipArray *arr,*arr1; + + HIPCHECK(hipMalloc3DArray(&arr, &channelDesc, make_hipExtent(width, height, depth), hipArrayDefault)); + HIPCHECK(hipMalloc3DArray(&arr1, &channelDesc, make_hipExtent(width, height, depth), hipArrayDefault)); + hipMemcpy3DParms myparms = {0}; + myparms.srcPos = make_hipPos(0,0,0); + myparms.dstPos = make_hipPos(0,0,0); + myparms.srcPtr = make_hipPitchedPtr(hData, width * sizeof(T), width, height); + myparms.dstArray = arr; + myparms.extent = make_hipExtent(width , height, depth); +#ifdef __HIP_PLATFORM_NVCC__ + myparms.kind = cudaMemcpyHostToDevice; +#else + myparms.kind = hipMemcpyHostToDevice; +#endif + HIPCHECK(hipMemcpy3D(&myparms)); + HIPCHECK(hipDeviceSynchronize()); + //Array to Array + memset(&myparms,0x0, sizeof(hipMemcpy3DParms)); + myparms.srcPos = make_hipPos(0,0,0); + myparms.dstPos = make_hipPos(0,0,0); + myparms.srcArray = arr; + myparms.dstArray = arr1; + myparms.extent = make_hipExtent(width, height, depth); +#ifdef __HIP_PLATFORM_NVCC__ + myparms.kind = cudaMemcpyDeviceToDevice; +#else + myparms.kind = hipMemcpyDeviceToDevice; +#endif + HIPCHECK(hipMemcpy3D(&myparms)); + HIPCHECK(hipDeviceSynchronize()); + + T *hOutputData = (T*) malloc(size); + memset(hOutputData, 0, size); + //Device to host + memset(&myparms,0x0, sizeof(hipMemcpy3DParms)); + myparms.srcPos = make_hipPos(0,0,0); + myparms.dstPos = make_hipPos(0,0,0); + myparms.dstPtr = make_hipPitchedPtr(hOutputData, width * sizeof(T), width, height); + myparms.srcArray = arr1; + myparms.extent = make_hipExtent(width, height, depth); +#ifdef __HIP_PLATFORM_NVCC__ + myparms.kind = cudaMemcpyDeviceToHost; +#else + myparms.kind = hipMemcpyDeviceToHost; +#endif + HIPCHECK(hipMemcpy3D(&myparms)); + HIPCHECK(hipDeviceSynchronize()); + + // Check result + HipTest::checkArray(hData,hOutputData,width,height,depth); + hipFreeArray(arr); + hipFreeArray(arr1); + free(hData); + free(hOutputData); +} + +int main(int argc, char **argv) +{ + for(int i=1;i<25;i++) + { + runTest(i,i,i, hipChannelFormatKindFloat); + runTest(i+1,i,i, hipChannelFormatKindSigned); + runTest(i,i+1,i, hipChannelFormatKindSigned); + } + passed(); +} diff --git a/tests/src/texture/simpleTexture2DLayered.cpp b/tests/src/texture/simpleTexture2DLayered.cpp index fa545cb4ca..e5014dae6b 100644 --- a/tests/src/texture/simpleTexture2DLayered.cpp +++ b/tests/src/texture/simpleTexture2DLayered.cpp @@ -65,7 +65,7 @@ void runTest(int width,int height,int num_layers,texture texc; template -__global__ void simpleKernel3DArray(T* outputData, +__global__ void simpleKernel3DArray(T* outputData, int width, int height,int depth) { @@ -55,21 +52,18 @@ __global__ void simpleKernel3DArray(T* outputData, for (int j = 0; j < height; j++) { for (int k = 0; k < width; k++) { if(std::is_same::value) - outputData[i*width*height + j*width + k] = tex3D(texf, texf.textureObject, k, j, i); + outputData[i*width*height + j*width + k] = tex3D(texf, k, j, i); else if(std::is_same::value) - outputData[i*width*height + j*width + k] = tex3D(texi, texi.textureObject, k, j, i); + outputData[i*width*height + j*width + k] = tex3D(texi, k, j, i); else if(std::is_same::value) - outputData[i*width*height + j*width + k] = tex3D(texc, texc.textureObject, k, j, i); + outputData[i*width*height + j*width + k] = tex3D(texc, k, j, i); } } } } -//////////////////////////////////////////////////////////////////////////////// -//! Run a simple test for tex3D -//////////////////////////////////////////////////////////////////////////////// template -void runTest(int width,int height,int depth,texture *tex) +void runTest(int width,int height,int depth,texture *tex, hipChannelFormatKind formatKind) { unsigned int size = width * height * depth * sizeof(T); T* hData = (T*) malloc(size); @@ -84,17 +78,21 @@ void runTest(int width,int height,int depth,texture(i,i,i,&texf); - runTest(i+1,i,i,&texi); - runTest(i,i+1,i,&texc); + runTest(i,i,i,&texf, hipChannelFormatKindFloat); + runTest(i+1,i,i,&texi, hipChannelFormatKindSigned); + runTest(i,i+1,i,&texc, hipChannelFormatKindSigned); } passed(); } - From 92cc29ae2b5c2f0e0542e12b8747ce77f3dc7ebd Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Mon, 17 Feb 2020 09:19:26 -0800 Subject: [PATCH 035/154] Let HIP-Clang inline all functions by default (#1875) This is a quick workaround to match HCC behavior for performance since inlining usually results in more optimization opportunities therefore better performance. We will fine tuning inline threashold later. --- bin/hipcc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/hipcc b/bin/hipcc index 1bdb466c68..41f11a36b4 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -727,6 +727,10 @@ if ($HIP_PLATFORM eq "clang") { $HIPCXXFLAGS .= " -O3"; $HIPLDFLAGS .= " -O3"; } + if ($optArg ne "-O0") { + $HIPCXXFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; + $HIPLDFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; + } $HIP_DEVLIB_FLAGS = " --hip-device-lib-path=$DEVICE_LIB_PATH"; $HIPCXXFLAGS .= " $HIP_DEVLIB_FLAGS"; if (not $isWindows) { From 9b4f39e1d8f59ee2d3aea3de63f1d2fdb59c7dd0 Mon Sep 17 00:00:00 2001 From: Alex Voicu Date: Tue, 18 Feb 2020 17:20:27 +0200 Subject: [PATCH 036/154] Tweak synchronous memcpy implementation (#1809) The existing one can have issues on certain systems, therefore this limits use of direct memcpy via largeBAR to sizes where it is unequivocally better. Also addresses SWDEV-220030 and SWDEV-222237. --- src/hip_memory.cpp | 105 +++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 62 deletions(-) diff --git a/src/hip_memory.cpp b/src/hip_memory.cpp index d965059923..1bcf10f982 100644 --- a/src/hip_memory.cpp +++ b/src/hip_memory.cpp @@ -37,7 +37,6 @@ __device__ uint32_t __hip_device_page_flag[__HIP_NUM_PAGES]; namespace hip_internal { namespace { - inline const char* hsa_to_string(hsa_status_t err) noexcept { @@ -149,13 +148,14 @@ namespace { const_cast(p), &r, nullptr, nullptr, nullptr), __FILE__, __func__, __LINE__); - r.size = is_large_BAR || (type(r.agentOwner) == HSA_DEVICE_TYPE_CPU) ? - UINT32_MAX : sizeof(hsa_amd_pointer_info_t); + if (is_large_BAR) r.size = UINT32_MAX; + else if (type(r.agentOwner) == HSA_DEVICE_TYPE_CPU) r.size = INT32_MAX; return r; } - constexpr size_t staging_sz{4 * 1024 * 1024}; // 2 Pages. + constexpr size_t staging_sz{4 * 1024 * 1024}; // 2 Pages. + constexpr size_t max_std_memcpy_sz{8 * 1024}; // 8 KiB. thread_local const std::unique_ptr staging_buffer{ []() { @@ -202,8 +202,8 @@ namespace { } // Unnamed namespace. inline -void do_copy(void* __restrict dst, const void* __restrict src, std::size_t n, - hsa_agent_t da, hsa_agent_t sa) { +void do_copy(void* __restrict dst, const void* __restrict src, size_t n, + hsa_agent_t da, hsa_agent_t sa) { hsa_signal_silent_store_relaxed(copy_signal, 1); throwing_result_check( hsa_amd_memory_async_copy(dst, da, src, sa, n, 0, nullptr, copy_signal), @@ -224,10 +224,10 @@ void do_std_memcpy( inline void d2h_copy(void* __restrict dst, const void* __restrict src, size_t n, hsa_amd_pointer_info_t si) { - // TODO: characterise direct largeBAR reads from agent-allocated memory. - // if (si.size == UINT32_MAX) { - // return do_std_memcpy(dst, src, n); - // } + if (si.size == INT32_MAX) return do_std_memcpy(dst, src, n); + if (si.size == UINT32_MAX && n <= max_std_memcpy_sz) { + return do_std_memcpy(dst, src, n); + } const auto di{info(dst)}; @@ -256,7 +256,8 @@ void d2h_copy(void* __restrict dst, const void* __restrict src, size_t n, inline void h2d_copy(void* __restrict dst, const void* __restrict src, size_t n, hsa_amd_pointer_info_t di) { - if (di.size == UINT32_MAX) { + if (di.size == INT32_MAX) return do_std_memcpy(dst, src, n); + if (di.size == UINT32_MAX && n <= max_std_memcpy_sz) { return do_std_memcpy(dst, src, n); } @@ -264,8 +265,8 @@ void h2d_copy(void* __restrict dst, const void* __restrict src, size_t n, if (si.type == HSA_EXT_POINTER_TYPE_LOCKED) { src = static_cast(si.agentBaseAddress) + - (static_cast(src) - - static_cast(si.hostBaseAddress)); + (static_cast(src) - + static_cast(si.hostBaseAddress)); do_copy(dst, src, n, di.agentOwner, di.agentOwner); } else if (n <= staging_sz) { @@ -288,53 +289,30 @@ void h2d_copy(void* __restrict dst, const void* __restrict src, size_t n, inline void generic_copy(void* __restrict dst, const void* __restrict src, size_t n, hsa_amd_pointer_info_t di, hsa_amd_pointer_info_t si) { - if (di.size == UINT32_MAX && si.size == UINT32_MAX) { + if (di.size == INT32_MAX && si.size == INT32_MAX) { + return do_std_memcpy(dst, src, n); + } + if (di.size == UINT32_MAX && si.size == UINT32_MAX && + n <= max_std_memcpy_sz) { return do_std_memcpy(dst, src, n); } - std::unique_ptr lck0{ - nullptr, [](void* p) { hsa_amd_memory_unlock(p); }}; - std::unique_ptr lck1{nullptr, lck0.get_deleter()}; - - switch (si.type) { - case HSA_EXT_POINTER_TYPE_HSA: - if (di.type == HSA_EXT_POINTER_TYPE_HSA) { - hsa_memory_copy(dst, src, n); - return; // TODO: do_copy(dst, src, n, di.agentOwner, si.agentOwner); + switch (type(si.agentOwner)) { + case HSA_DEVICE_TYPE_GPU: + if (type(di.agentOwner) == HSA_DEVICE_TYPE_GPU) { + throwing_result_check( + hsa_amd_agents_allow_access( + 1u, &si.agentOwner, nullptr, di.agentBaseAddress), + __FILE__, __func__, __LINE__); + return do_copy(dst, src, n, di.agentOwner, si.agentOwner); } - - if (di.type == HSA_EXT_POINTER_TYPE_UNKNOWN || - di.type == HSA_EXT_POINTER_TYPE_LOCKED) { - return d2h_copy(dst, src, n, si); - } - break; - case HSA_EXT_POINTER_TYPE_LOCKED: - if (di.type == HSA_EXT_POINTER_TYPE_UNKNOWN) { - std::memcpy(dst, si.hostBaseAddress, n); - - return; - } - if (di.type == HSA_EXT_POINTER_TYPE_LOCKED) { - std::memcpy(di.hostBaseAddress, si.hostBaseAddress, n); - - return; - } - src = si.agentBaseAddress; - si.agentOwner = di.agentOwner; - break; - case HSA_EXT_POINTER_TYPE_UNKNOWN: - if (di.type == HSA_EXT_POINTER_TYPE_UNKNOWN) { - std::memcpy(dst, src, n); - - return; - } - if (di.type == HSA_EXT_POINTER_TYPE_LOCKED) { - std::memcpy(di.hostBaseAddress, src, n); - - return; + return d2h_copy(dst, src, n, si); + case HSA_DEVICE_TYPE_CPU: + if (type(di.agentOwner) == HSA_DEVICE_TYPE_CPU) { + return do_std_memcpy(dst, src, n); } return h2d_copy(dst, src, n, di); - default: do_copy(dst, src, n, di.agentOwner, si.agentOwner); break; + default: throw std::runtime_error{"Unsupported copy type."}; } } @@ -343,14 +321,17 @@ void memcpy_impl(void* __restrict dst, const void* __restrict src, size_t n, hipMemcpyKind k) noexcept { switch (k) { case hipMemcpyHostToHost: std::memcpy(dst, src, n); break; - case hipMemcpyHostToDevice: - return is_large_BAR ? do_std_memcpy(dst, src, n) - : h2d_copy(dst, src, n, info(dst)); - case hipMemcpyDeviceToHost: - // TODO: characterise direct largeBAR reads from agent-allocated memory. - return /*is_large_BAR ? do_std_memcpy(dst, src, n) - : */d2h_copy(dst, src, n, info(src)); - case hipMemcpyDeviceToDevice: hsa_memory_copy(dst, src, n); break; + case hipMemcpyHostToDevice: return h2d_copy(dst, src, n, info(dst)); + case hipMemcpyDeviceToHost: return d2h_copy(dst, src, n, info(src)); + case hipMemcpyDeviceToDevice: { + const auto di{info(dst)}; + const auto si{info(src)}; + throwing_result_check( + hsa_amd_agents_allow_access( + 1u, &si.agentOwner, nullptr, di.agentBaseAddress), + __FILE__, __func__, __LINE__); + return do_copy(dst, src, n, di.agentOwner, si.agentOwner); + } default: return generic_copy(dst, src, n, info(dst), info(src)); } } From 82ec3c1c5b5979147bef1d71a73f910d82d7510c Mon Sep 17 00:00:00 2001 From: eshcherb <33529668+eshcherb@users.noreply.github.com> Date: Wed, 19 Feb 2020 02:17:49 -0600 Subject: [PATCH 037/154] adding hipExtModuleLaunchKernel to tracing layer (#1880) --- hip_prof_gen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hip_prof_gen.py b/hip_prof_gen.py index 9e90c1558c..d2da7cd4df 100755 --- a/hip_prof_gen.py +++ b/hip_prof_gen.py @@ -447,7 +447,8 @@ if len(sys.argv) > 3: OUTPUT = sys.argv[3] # API declaration map api_map = { - 'hipHccModuleLaunchKernel': '' + 'hipHccModuleLaunchKernel': '', + 'hipExtModuleLaunchKernel': '' } # API options map opts_map = {} From c1a70707e031a9b88d4019e865032e7fb16bcda4 Mon Sep 17 00:00:00 2001 From: Sarbojit2019 <52527887+SarbojitAMD@users.noreply.github.com> Date: Wed, 19 Feb 2020 13:48:20 +0530 Subject: [PATCH 038/154] [HIPIFY] Add back missing execute permission to hipify-perl (#1881) hipify-perl script lost its executable permission hence "samples/0_Intro/square" was failing. Fixes SWDEV 223433. --- bin/hipify-perl | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/hipify-perl diff --git a/bin/hipify-perl b/bin/hipify-perl old mode 100644 new mode 100755 From 54447268cc1115fc84305c0a5327bc0ac5a12c5c Mon Sep 17 00:00:00 2001 From: Jatin Chaudhary Date: Fri, 21 Feb 2020 21:52:19 +0530 Subject: [PATCH 039/154] Generating hiprtc lib with hcc+hip-clang Review comments - generate hiprtc lib everytime when HIP_PLATFORM is hcc Changes for hip-clang Removing pre processor directive to simplify Change-Id: Id38ab368362b58ee0458baeb8051fea709ae6bba --- CMakeLists.txt | 24 +++++++++--------------- src/hiprtc.cpp | 4 ++-- tests/src/hiprtc/saxpy.cpp | 2 +- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b6c4adaa09..f64ca45559 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -373,18 +373,17 @@ if(HIP_PLATFORM STREQUAL "hcc") set_property ( TARGET hip_hcc PROPERTY VERSION "${HIP_LIB_VERSION_STRING}" ) set_property ( TARGET hip_hcc PROPERTY SOVERSION "${HIP_LIB_VERSION_MAJOR}" ) - if(HIP_COMPILER STREQUAL "hcc") - target_link_libraries(hip_hcc PRIVATE hc_am) - target_link_libraries(hip_hcc_static PRIVATE hc_am) + target_link_libraries(hip_hcc PRIVATE hc_am) + target_link_libraries(hip_hcc_static PRIVATE hc_am) - add_library(hiprtc SHARED src/hiprtc.cpp src/code_object_bundle.cpp) - set_property ( TARGET hiprtc PROPERTY VERSION "${HIP_LIB_VERSION_STRING}" ) - set_property ( TARGET hiprtc PROPERTY SOVERSION "${HIP_LIB_VERSION_MAJOR}" ) + add_library(hiprtc SHARED src/hiprtc.cpp src/code_object_bundle.cpp) + set_property ( TARGET hiprtc PROPERTY VERSION "${HIP_LIB_VERSION_STRING}" ) + set_property ( TARGET hiprtc PROPERTY SOVERSION "${HIP_LIB_VERSION_MAJOR}" ) + + target_include_directories( + hiprtc SYSTEM + PRIVATE ${PROJECT_SOURCE_DIR}/include ${HSA_PATH}/include) - target_include_directories( - hiprtc SYSTEM - PRIVATE ${PROJECT_SOURCE_DIR}/include ${HSA_PATH}/include) -endif() set_target_properties(hip_hcc PROPERTIES CXX_VISIBILITY_PRESET hidden) set_target_properties(hip_hcc PROPERTIES VISIBILITY_INLINES_HIDDEN 1) @@ -448,12 +447,7 @@ endif() ############################# # Install hip_hcc if platform is hcc if(HIP_PLATFORM STREQUAL "hcc") - if(HIP_COMPILER STREQUAL "hcc") install(TARGETS hip_hcc_static hip_hcc hiprtc DESTINATION lib) - else() - install(TARGETS hip_hcc_static hip_hcc DESTINATION lib) - endif() - endif() # Install .hipInfo diff --git a/src/hiprtc.cpp b/src/hiprtc.cpp index 5198bf0cbb..8dcb944f72 100644 --- a/src/hiprtc.cpp +++ b/src/hiprtc.cpp @@ -235,7 +235,7 @@ struct _hiprtcProgram { const auto it{find_if(reader.sections.begin(), reader.sections.end(), [](const section* x) { - return x->get_name() == ".kernel"; + return (x->get_name() == ".hip_fatbin") || (x->get_name() == ".kernel"); })}; if (it == reader.sections.end()) return false; @@ -513,7 +513,7 @@ hiprtcResult hiprtcCompileProgram(hiprtcProgram p, int n, const char** o) const auto src{p->writeTemporaryFiles(tmp.path())}; - vector args{hipcc, "-shared"}; + vector args{hipcc, "-fPIC -shared"}; if (n) args.insert(args.cend(), o, o + n); handleTarget(args); diff --git a/tests/src/hiprtc/saxpy.cpp b/tests/src/hiprtc/saxpy.cpp index 156a44afe4..d063578757 100755 --- a/tests/src/hiprtc/saxpy.cpp +++ b/tests/src/hiprtc/saxpy.cpp @@ -143,7 +143,7 @@ int main() hipMemcpyDtoH(hOut.get(), dOut, bufferSize); for (size_t i = 0; i < n; ++i) { - if (a * hX[i] + hY[i] != hOut[i]) { failed("Validation failed."); } + if (fabs(a * hX[i] + hY[i] - hOut[i]) > fabs(hOut[i])* 1e-6) { failed("Validation failed."); } } hipFree(dX); From c6bb103dfff8b83cfcbd64f4fbbbeb80bc0a8a64 Mon Sep 17 00:00:00 2001 From: Payam Date: Mon, 24 Feb 2020 13:59:23 -0500 Subject: [PATCH 040/154] updated lib elf path and clean ups Change-Id: Id0b3c295fa8353a5da8517204bf53dab9887defb --- vdi/CMakeLists.txt | 70 +++++++++------------------------------------- 1 file changed, 13 insertions(+), 57 deletions(-) diff --git a/vdi/CMakeLists.txt b/vdi/CMakeLists.txt index 8dc6d4f6f7..0140408875 100644 --- a/vdi/CMakeLists.txt +++ b/vdi/CMakeLists.txt @@ -19,14 +19,8 @@ if(CMAKE_BUILD_TYPE MATCHES "^Debug$") add_definitions(-DDEBUG) endif() -#command example: -#cmake -DVDI_DIR=/home/pghafari/vdi/vdi -DLIBOCL_STATIC_DIR=/home/pghafari/git-ocl/build/amdocl -DLIBVDI_STATIC_DIR=/home/pghafari/vdi/vdi/build -DUSE_COMGR_LIBRARY=yes -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH="/home/pghafari/rocmgitvdi/hsa-runtime/opensrc/hsa-runtime/build;/home/pghafari/rocmgitvdi/hsa-runtime/opensrc;/home/pghafari/rocmgitvdi/support/lib/comgr;/home/pghafari/rocmgitvdi/support/lib/comgr/build;" -DCMAKE_MODULE_PATH=/home/pghafari/rocmgit/opencl/cmake/modules -DLLVM_INCLUDES=/home/pghafari/rocmgit/llvm-project/llvm/include .. - set(USE_PROF_API "1") -find_package(ROCT REQUIRED) -find_package(ROCR REQUIRED) - if(NOT DEFINED LIBVDI_STATIC_DIR) find_path(LIBVDI_STATIC_DIR NAMES libamdvdi_static.a @@ -42,7 +36,7 @@ if(NOT DEFINED VDI_DIR) PATHS /opt/rocm/vdi ) endif() -message("vdilib:${LIBVDI_STATIC_DIR} ,found vdi includes ${VDI_DIR}") +message("Found Static vdi lib:${LIBVDI_STATIC_DIR} and vdi includes: ${VDI_DIR}") set(PROF_API_HEADER_PATH ${VDI_DIR}/platform) ############################# # Profiling API support @@ -56,7 +50,7 @@ set(PROF_API_LOG "${PROJECT_BINARY_DIR}/hip_prof_gen.log.txt") set(PROF_API_CMD "${PROF_API_GEN} -v -t --priv ${OPT_PROF_API} ${PROF_API_HDR} ${PROF_API_SRC} ${PROF_API_STR} >${PROF_API_LOG}") MESSAGE(STATUS "Generating profiling promitives: ${PROF_API_STR}") execute_process(COMMAND sh -c "rm -f ${PROF_API_STR}; ${PROF_API_CMD}") -MESSAGE(COMMAND sh -c "rm -f ${PROF_API_STR}; ${PROF_API_CMD}") +#MESSAGE(COMMAND sh -c "rm -f ${PROF_API_STR}; ${PROF_API_CMD}") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PROF_API_GEN} ${PROF_API_HDR} ${PROF_API_STR}) # Enable profiling API @@ -92,7 +86,6 @@ if (DEFINED LLVM_INCLUDES AND NOT ${LLVM_INCLUDES} STREQUAL "") include_directories(${LLVM_INCLUDES}) endif() # if (DEFINED LLVM_INCLUDES AND NOT ${LLVM_INCLUDES} STREQUAL "") -#find_package(hsa-runtime REQUIRED CONFIG PATHS ${HSA_RUNTIME} NO_DEFAULT_PATH) include_directories(${CMAKE_SOURCE_DIR}) include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${CMAKE_SOURCE_DIR}/elfio) @@ -102,9 +95,8 @@ include_directories(${VDI_DIR}) include_directories(${VDI_DIR}/include) include_directories(${VDI_DIR}/compiler/lib) include_directories(${VDI_DIR}/compiler/lib/include) -include_directories(${VDI_DIR}/compiler/lib/loaders) -include_directories(${VDI_DIR}/compiler/lib/loaders/elf/utils/common) -include_directories(${VDI_DIR}/compiler/lib/loaders/elf/utils/libelf) +include_directories(${VDI_DIR}/elf/utils/common) +include_directories(${VDI_DIR}/elf/utils/libelf) add_definitions(-DUSE_COMGR_LIBRARY -DCOMGR_DYN_DLL) find_package(amd_comgr REQUIRED CONFIG PATHS @@ -115,12 +107,9 @@ add_definitions(-DUSE_COMGR_LIBRARY -DCOMGR_DYN_DLL) ) MESSAGE(STATUS "Code Object Manager found at ${amd_comgr_DIR}.") -#find_package(amd_comgr REQUIRED CONFIG) include_directories("$") add_definitions(-DBSD_LIBELF) -#find_package( Threads REQUIRED CONFIG) -#find_package( LibElf REQUIRED CONFIG) add_library(amdhip64 SHARED hip_context.cpp @@ -143,21 +132,17 @@ add_library(amdhip64 SHARED cl_lqdflash_amd.cpp fixme.cpp ) -add_library(amdvdi_static STATIC IMPORTED) -set_target_properties(amdvdi_static PROPERTIES IMPORTED_LOCATION "${LIBVDI_STATIC_DIR}/libamdvdi_static.a") -#add_library(amdocl64 SHARED IMPORTED) -#set_target_properties(amdocl64 PROPERTIES IMPORTED_LOCATION "/opt/rocm/opencl/lib/x86_64/libamdocl64.so") -#add_library(oclelf STATIC IMPORTED) -#set_target_properties(oclelf PROPERTIES IMPORTED_LOCATION "${LIBOCLELF_STATIC_DIR}/liboclelf.a") - add_library(host INTERFACE) - target_link_libraries(host INTERFACE amdhip64) - add_library(device INTERFACE) - target_link_libraries(device INTERFACE host) - #target_link_libraries(amdhip64 PRIVATE amd_comgr) +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) +include(${LIBVDI_STATIC_DIR}/amdvdi_staticTargets.cmake) -target_link_libraries(amdhip64 amdvdi_static pthread dl ${ROCT_LIBRARIES} ${ROCR_LIBRARIES}) -#target_link_libraries(amdhip64 amdvdi_static amdocl64_static pthread dl ${ROCT_LIBRARIES} ${ROCR_LIBRARIES}) +add_library(host INTERFACE) +target_link_libraries(host INTERFACE amdhip64) +add_library(device INTERFACE) +target_link_libraries(device INTERFACE host) + +target_link_libraries(amdhip64 amdvdi_static pthread dl) #install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink libamdhip64.so lib/libhip_hcc.so)" ) @@ -172,32 +157,3 @@ INSTALL(FILES ${CMAKE_BINARY_DIR}/lib/libhiprtc.so DESTINATION lib COMPONENT MAI INSTALL(TARGETS amdhip64 host device EXPORT hip-targets DESTINATION ${LIB_INSTALL_DIR}) INSTALL(EXPORT hip-targets DESTINATION ${CONFIG_PACKAGE_INSTALL_DIR} NAMESPACE hip::) -if(0) -set(CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators") - -set(CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc.") -set(CPACK_PACKAGE_VENDOR "AMD") -set(CPACK_PACKAGING_INSTALL_PREFIX "/opt/rocm/hip-on-vdi") - -set(CPACK_PACKAGE_VERSION_MAJOR "1") -set(CPACK_PACKAGE_VERSION_MINOR "0") -set(CPACK_PACKAGE_VERSION_PATCH "0") - -# Debian CPACK variables -set(CPACK_DEB_COMPONENT_INSTALL ON) - -set(CPACK_DEBIAN_MAIN_FILE_NAME "hip-on-vdi-1.0.0.deb") -set(CPACK_DEBIAN_MAIN_PACKAGE_NAME "hip-on-vdi") -#set(CPACK_DEBIAN_MAIN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEB/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEB/prerm") - -# RPM CPACK variables -set(CPACK_RPM_COMPONENT_INSTALL ON) - -set(CPACK_RPM_MAIN_FILE_NAME "hip-on-vdi-1.0.0.rpm") -set(CPACK_RPM_MAIN_PACKAGE_NAME "hip-on-vdi") -#set(CPACK_RPM_MAIN_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post") -#set(CPACK_RPM_MAIN_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun") - -set(CPACK_COMPONENTS_ALL MAIN) -include(CPack) -endif() From c22eb7808ddf3fe5a7f61553062f90628de9120f Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Sun, 23 Feb 2020 15:34:31 -0500 Subject: [PATCH 041/154] HIP-VDI texture rework The current texture implementation is based off the one for HIP-HCC. There's a lot of problems with it - only creating images from buffers, hard coding logic and ignoring user parameters. This leads to a whole lot of UB even with simple examples (as seen with RedShift's code). This CL is aimed to bring the HIP-VDI texture implementation closer to what is described by Cuda. hipMemcpyAtoA() - image to image copy. hipMemcpyHtoA()/hipMemcpyDtoA() - buffer to image copy. hipMemcpyAtoH()/hipMemcpyAtoD() - image to buffer copy. hipArrayCreate()/hipArray3DCreate()/hipMallocArray()/hipMalloc3DArray() - creates 1D/2D/3D/1D Array/2D Array images. hipCreateTextureObject() - creates sampler, (optional) creates 1D/2D image from buffer, (optional) creates image views. hipBindTexture() - creates 1D image from buffer (should create a typed buffer, however this is not compatible with HIP-HCC). hipBindTexture2D() - creates 2D image form buffer. hipBindTextureToArray() - creates image view. hipTexRefSetAddress() - creates 1D image from buffer (should create a typed buffer, however this is not compatible with HIP-HCC). hipTexRefSetAddress2D() - creates 2D image from buffer. hipTexRefSetArray() - creates image view. There are still a lot of TODOs in the code, here's a few important ones: 1. VDI doesn't support a lot of sampler flags. 2. VDI doesn't support device to image 2D/3D copy. 3. Mipmaps implementation is incomplete. 4. Image view implementation is incomplete. Change-Id: Ia374ee27aa14f76451fee7667495036f4419a487 --- vdi/hip_conversions.hpp | 583 +++++++++++++ vdi/hip_memory.cpp | 1833 +++++++++++++++++++++++---------------- vdi/hip_texture.cpp | 1745 ++++++++++++++++++++++--------------- 3 files changed, 2718 insertions(+), 1443 deletions(-) create mode 100644 vdi/hip_conversions.hpp diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp new file mode 100644 index 0000000000..321ef7e024 --- /dev/null +++ b/vdi/hip_conversions.hpp @@ -0,0 +1,583 @@ +/* +Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#pragma once + +#include +#include + +namespace hip +{ +inline +cl_channel_type getCLChannelType(const hipArray_Format hipFormat, + const hipTextureReadMode hipReadMode) { + if (hipReadMode == hipReadModeElementType) { + switch (hipFormat) { + case HIP_AD_FORMAT_UNSIGNED_INT8: + return CL_UNSIGNED_INT8; + case HIP_AD_FORMAT_SIGNED_INT8: + return CL_SIGNED_INT8; + case HIP_AD_FORMAT_UNSIGNED_INT16: + return CL_UNSIGNED_INT16; + case HIP_AD_FORMAT_SIGNED_INT16: + return CL_SIGNED_INT16; + case HIP_AD_FORMAT_UNSIGNED_INT32: + return CL_UNSIGNED_INT32; + case HIP_AD_FORMAT_SIGNED_INT32: + return CL_SIGNED_INT32; + case HIP_AD_FORMAT_HALF: + return CL_HALF_FLOAT; + case HIP_AD_FORMAT_FLOAT: + return CL_FLOAT; + } + } else if (hipReadMode == hipReadModeNormalizedFloat) { + switch (hipFormat) { + case HIP_AD_FORMAT_UNSIGNED_INT8: + return CL_UNORM_INT8; + case HIP_AD_FORMAT_SIGNED_INT8: + return CL_SNORM_INT8; + case HIP_AD_FORMAT_UNSIGNED_INT16: + return CL_UNORM_INT16; + case HIP_AD_FORMAT_SIGNED_INT16: + return CL_SNORM_INT16; + case HIP_AD_FORMAT_UNSIGNED_INT32: + return CL_UNSIGNED_INT32; + case HIP_AD_FORMAT_SIGNED_INT32: + return CL_SIGNED_INT32; + case HIP_AD_FORMAT_HALF: + return CL_HALF_FLOAT; + case HIP_AD_FORMAT_FLOAT: + return CL_FLOAT; + } + } + + ShouldNotReachHere(); +} + +inline +cl_channel_order getCLChannelOrder(const unsigned int hipNumChannels) { + switch (hipNumChannels) { + case 1: + return CL_R; + case 2: + return CL_RG; + case 4: + return CL_RGBA; + } + + ShouldNotReachHere(); +} + +inline +cl_mem_object_type getCLMemObjectType(const unsigned int hipWidth, + const unsigned int hipHeight, + const unsigned int hipDepth, + const unsigned int flags) { + if (flags == hipArrayDefault) { + if ((hipWidth != 0) && (hipHeight == 0) && (hipDepth == 0)) { + return CL_MEM_OBJECT_IMAGE1D; + } else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth == 0)) { + return CL_MEM_OBJECT_IMAGE2D; + } else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth != 0)) { + return CL_MEM_OBJECT_IMAGE3D; + } + } else if (flags == hipArrayLayered) { + if ((hipWidth != 0) && (hipHeight == 0) && (hipDepth != 0)) { + return CL_MEM_OBJECT_IMAGE1D_ARRAY; + } else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth != 0)) { + return CL_MEM_OBJECT_IMAGE2D_ARRAY; + } + } + + ShouldNotReachHere(); +} + +inline +cl_addressing_mode getCLAddressingMode(const hipTextureAddressMode hipAddressMode) { + switch (hipAddressMode) { + case hipAddressModeWrap: + return CL_ADDRESS_REPEAT; + case hipAddressModeClamp: + return CL_ADDRESS_CLAMP; + case hipAddressModeMirror: + return CL_ADDRESS_MIRRORED_REPEAT; + case hipAddressModeBorder: + return CL_ADDRESS_CLAMP_TO_EDGE; + } + + ShouldNotReachHere(); +} + +inline +cl_filter_mode getCLFilterMode(const hipTextureFilterMode hipFilterMode) { + switch (hipFilterMode) { + case hipFilterModePoint: + return CL_FILTER_NEAREST; + case hipFilterModeLinear: + return CL_FILTER_LINEAR; + } + + ShouldNotReachHere(); +} + +inline +cl_mem_object_type getCLMemObjectType(const hipResourceType hipResType) { + switch (hipResType) { + case hipResourceTypeLinear: + return CL_MEM_OBJECT_IMAGE1D; + case hipResourceTypePitch2D: + return CL_MEM_OBJECT_IMAGE2D; + } + + ShouldNotReachHere(); +} + +inline +size_t getElementSize(const hipArray_Format arrayFormat) { + switch (arrayFormat) { + case HIP_AD_FORMAT_UNSIGNED_INT8: + case HIP_AD_FORMAT_SIGNED_INT8: + return 1; + case HIP_AD_FORMAT_UNSIGNED_INT16: + case HIP_AD_FORMAT_SIGNED_INT16: + case HIP_AD_FORMAT_HALF: + return 2; + case HIP_AD_FORMAT_UNSIGNED_INT32: + case HIP_AD_FORMAT_SIGNED_INT32: + case HIP_AD_FORMAT_FLOAT: + return 4; + } + + ShouldNotReachHere(); +} + +inline +hipChannelFormatDesc getChannelFormatDesc(int numChannels, + hipArray_Format arrayFormat) { + switch (arrayFormat) { + case HIP_AD_FORMAT_UNSIGNED_INT8: + switch (numChannels) { + case 1: + return {8, 0, 0, 0, hipChannelFormatKindUnsigned}; + case 2: + return {8, 8, 0, 0, hipChannelFormatKindUnsigned}; + case 4: + return {8, 8, 8, 8, hipChannelFormatKindUnsigned}; + } + case HIP_AD_FORMAT_SIGNED_INT8: + switch (numChannels) { + case 1: + return {8, 0, 0, 0, hipChannelFormatKindSigned}; + case 2: + return {8, 8, 0, 0, hipChannelFormatKindSigned}; + case 4: + return {8, 8, 8, 8, hipChannelFormatKindSigned}; + } + case HIP_AD_FORMAT_UNSIGNED_INT16: + switch (numChannels) { + case 1: + return {16, 0, 0, 0, hipChannelFormatKindUnsigned}; + case 2: + return {16, 16, 0, 0, hipChannelFormatKindUnsigned}; + case 4: + return {16, 16, 16, 16, hipChannelFormatKindUnsigned}; + } + case HIP_AD_FORMAT_SIGNED_INT16: + switch (numChannels) { + case 1: + return {16, 0, 0, 0, hipChannelFormatKindSigned}; + case 2: + return {16, 16, 0, 0, hipChannelFormatKindSigned}; + case 4: + return {16, 16, 16, 16, hipChannelFormatKindSigned}; + } + case HIP_AD_FORMAT_UNSIGNED_INT32: + switch (numChannels) { + case 1: + return {32, 0, 0, 0, hipChannelFormatKindUnsigned}; + case 2: + return {32, 32, 0, 0, hipChannelFormatKindUnsigned}; + case 4: + return {32, 32, 32, 32, hipChannelFormatKindUnsigned}; + } + case HIP_AD_FORMAT_SIGNED_INT32: + switch (numChannels) { + case 1: + return {32, 0, 0, 0, hipChannelFormatKindSigned}; + case 2: + return {32, 32, 0, 0, hipChannelFormatKindSigned}; + case 4: + return {32, 32, 32, 32, hipChannelFormatKindSigned}; + } + case HIP_AD_FORMAT_HALF: + switch (numChannels) { + case 1: + return {16, 0, 0, 0, hipChannelFormatKindFloat}; + case 2: + return {16, 16, 0, 0, hipChannelFormatKindFloat}; + case 4: + return {16, 16, 16, 16, hipChannelFormatKindFloat}; + } + case HIP_AD_FORMAT_FLOAT: + switch (numChannels) { + case 1: + return {32, 0, 0, 0, hipChannelFormatKindFloat}; + case 2: + return {32, 32, 0, 0, hipChannelFormatKindFloat}; + case 4: + return {32, 32, 32, 32, hipChannelFormatKindFloat}; + } + } + + ShouldNotReachHere(); +} + +inline +unsigned int getNumChannels(const hipChannelFormatDesc& desc) { + return ((desc.x != 0) + (desc.y != 0) + (desc.z != 0) + (desc.w != 0)); +} + +inline +hipArray_Format getArrayFormat(const hipChannelFormatDesc& desc) { + switch (desc.f) { + case hipChannelFormatKindUnsigned: + switch (desc.x) { + case 8: + return HIP_AD_FORMAT_UNSIGNED_INT8; + case 16: + return HIP_AD_FORMAT_UNSIGNED_INT16; + case 32: + return HIP_AD_FORMAT_UNSIGNED_INT32; + } + case hipChannelFormatKindSigned: + switch (desc.x) { + case 8: + return HIP_AD_FORMAT_SIGNED_INT8; + case 16: + return HIP_AD_FORMAT_SIGNED_INT16; + case 32: + return HIP_AD_FORMAT_SIGNED_INT32; + } + case hipChannelFormatKindFloat: + switch (desc.x) { + case 16: + return HIP_AD_FORMAT_HALF; + case 32: + return HIP_AD_FORMAT_FLOAT; + } + } + + ShouldNotReachHere(); +} + +inline +int getNumChannels(const hipResourceViewFormat hipFormat) { + switch (hipFormat) { + case hipResViewFormatUnsignedChar1: + case hipResViewFormatSignedChar1: + case hipResViewFormatUnsignedShort1: + case hipResViewFormatSignedShort1: + case hipResViewFormatUnsignedInt1: + case hipResViewFormatHalf1: + case hipResViewFormatFloat1: + return 1; + case hipResViewFormatUnsignedChar2: + case hipResViewFormatSignedChar2: + case hipResViewFormatUnsignedShort2: + case hipResViewFormatSignedShort2: + case hipResViewFormatUnsignedInt2: + case hipResViewFormatHalf2: + case hipResViewFormatFloat2: + return 2; + case hipResViewFormatUnsignedChar4: + case hipResViewFormatSignedChar4: + case hipResViewFormatUnsignedShort4: + case hipResViewFormatSignedShort4: + case hipResViewFormatUnsignedInt4: + case hipResViewFormatHalf4: + case hipResViewFormatFloat4: + return 4; + } + + ShouldNotReachHere(); +} + +inline +hipArray_Format getArrayFormat(const hipResourceViewFormat hipFormat) { + switch (hipFormat) { + case hipResViewFormatUnsignedChar1: + case hipResViewFormatUnsignedChar2: + case hipResViewFormatUnsignedChar4: + return HIP_AD_FORMAT_UNSIGNED_INT8; + case hipResViewFormatSignedChar1: + case hipResViewFormatSignedChar2: + case hipResViewFormatSignedChar4: + return HIP_AD_FORMAT_SIGNED_INT8; + case hipResViewFormatUnsignedShort1: + case hipResViewFormatUnsignedShort2: + case hipResViewFormatUnsignedShort4: + return HIP_AD_FORMAT_UNSIGNED_INT16; + case hipResViewFormatSignedShort1: + case hipResViewFormatSignedShort2: + case hipResViewFormatSignedShort4: + return HIP_AD_FORMAT_SIGNED_INT16; + case hipResViewFormatUnsignedInt1: + case hipResViewFormatUnsignedInt2: + case hipResViewFormatUnsignedInt4: + return HIP_AD_FORMAT_UNSIGNED_INT32; + case hipResViewFormatSignedInt1: + case hipResViewFormatSignedInt2: + case hipResViewFormatSignedInt4: + return HIP_AD_FORMAT_SIGNED_INT32; + case hipResViewFormatHalf1: + case hipResViewFormatHalf2: + case hipResViewFormatHalf4: + return HIP_AD_FORMAT_HALF; + case hipResViewFormatFloat1: + case hipResViewFormatFloat2: + case hipResViewFormatFloat4: + return HIP_AD_FORMAT_FLOAT; + } + + ShouldNotReachHere(); +} + +inline +hipResourceViewFormat getResourceViewFormat(const hipChannelFormatDesc& desc) { + switch (desc.f) { + case hipChannelFormatKindUnsigned: + switch (getNumChannels(desc)) { + case 1: + switch (desc.x) { + case 8: + return hipResViewFormatUnsignedChar1; + case 16: + return hipResViewFormatUnsignedShort1; + case 32: + return hipResViewFormatUnsignedInt1; + } + case 2: + switch (desc.x) { + case 8: + return hipResViewFormatUnsignedChar2; + case 16: + return hipResViewFormatUnsignedShort2; + case 32: + return hipResViewFormatUnsignedInt2; + } + case 4: + switch (desc.x) { + case 8: + return hipResViewFormatUnsignedChar4; + case 16: + return hipResViewFormatUnsignedShort4; + case 32: + return hipResViewFormatUnsignedInt4; + } + } + case hipChannelFormatKindSigned: + switch (getNumChannels(desc)) { + case 1: + switch (desc.x) { + case 8: + return hipResViewFormatSignedChar1; + case 16: + return hipResViewFormatSignedShort1; + case 32: + return hipResViewFormatSignedInt1; + } + case 2: + switch (desc.x) { + case 8: + return hipResViewFormatSignedChar2; + case 16: + return hipResViewFormatSignedShort2; + case 32: + return hipResViewFormatSignedInt2; + } + case 4: + switch (desc.x) { + case 8: + return hipResViewFormatSignedChar4; + case 16: + return hipResViewFormatSignedShort4; + case 32: + return hipResViewFormatSignedInt4; + } + } + case hipChannelFormatKindFloat: + switch (getNumChannels(desc)) { + case 1: + switch (desc.x) { + case 16: + return hipResViewFormatHalf1; + case 32: + return hipResViewFormatFloat1; + } + case 2: + switch (desc.x) { + case 16: + return hipResViewFormatHalf2; + case 32: + return hipResViewFormatFloat2; + } + case 4: + switch (desc.x) { + case 16: + return hipResViewFormatHalf4; + case 32: + return hipResViewFormatFloat4; + } + } + } + + ShouldNotReachHere(); +} + +inline +hipTextureReadMode getReadMode(unsigned int flags) { + if (flags & HIP_TRSF_READ_AS_INTEGER) { + return hipReadModeElementType; + } else { + return hipReadModeNormalizedFloat; + } +} + +inline +int getNormalizedCoords(unsigned int flags) { + if (flags & HIP_TRSF_NORMALIZED_COORDINATES) { + return 1; + } else { + return 0; + } +} + +inline +int getSRGB(unsigned int flags) { + if (flags & HIP_TRSF_SRGB) { + return 1; + } else { + return 0; + } +} + +inline +hipTextureDesc getTextureDesc(const textureReference* texRef, + const hipTextureReadMode readMode) { + hipTextureDesc texDesc = {}; + std::memcpy(texDesc.addressMode, texRef->addressMode, sizeof(texDesc.addressMode)); + texDesc.filterMode = texRef->filterMode; + texDesc.readMode = readMode; + texDesc.sRGB = texRef->sRGB; + texDesc.normalizedCoords = texRef->normalized; + texDesc.maxAnisotropy = texRef->maxAnisotropy; + texDesc.mipmapFilterMode = texRef->mipmapFilterMode; + texDesc.mipmapLevelBias = texRef->mipmapLevelBias; + texDesc.minMipmapLevelClamp = texRef->minMipmapLevelClamp; + texDesc.maxMipmapLevelClamp = texRef->maxMipmapLevelClamp; + + return texDesc; +} + +inline +hipResourceViewDesc getResourceViewDesc(hipArray_const_t array, + const hipResourceViewFormat format) { + hipResourceViewDesc resViewDesc = {}; + resViewDesc.format = format; + resViewDesc.width = array->width; + resViewDesc.height = array->height; + resViewDesc.depth = array->depth; + resViewDesc.firstMipmapLevel = 0; + resViewDesc.lastMipmapLevel = 0; + resViewDesc.firstLayer = 0; + resViewDesc.lastLayer = 0; /* TODO add hipArray::numLayers */ + + return resViewDesc; +} + +inline +hipResourceViewDesc getResourceViewDesc(hipMipmappedArray_const_t array, + const hipResourceViewFormat format) { + hipResourceViewDesc resViewDesc = {}; + resViewDesc.format = format; + resViewDesc.width = array->width; + resViewDesc.height = array->height; + resViewDesc.depth = array->depth; + resViewDesc.firstMipmapLevel = 0; + resViewDesc.lastMipmapLevel = 0; /* TODO add hipMipmappedArray::numMipLevels */ + resViewDesc.firstLayer = 0; + resViewDesc.lastLayer = 0; /* TODO add hipArray::numLayers */ + + return resViewDesc; +} + +inline +std::pair getMemoryType(const hipMemcpyKind kind) { + switch (kind) { + case hipMemcpyHostToHost: + return {hipMemoryTypeHost, hipMemoryTypeHost}; + case hipMemcpyHostToDevice: + return {hipMemoryTypeHost, hipMemoryTypeDevice}; + case hipMemcpyDeviceToHost: + return {hipMemoryTypeDevice, hipMemoryTypeHost}; + case hipMemcpyDeviceToDevice: + return {hipMemoryTypeDevice, hipMemoryTypeDevice}; + case hipMemcpyDefault: + return {hipMemoryTypeUnified, hipMemoryTypeUnified}; + } + + ShouldNotReachHere(); +} + +inline +hipMemcpy3DParms getMemcpy3DParms(const hip_Memcpy2D& desc2D) { + hipMemcpy3DParms desc3D = {}; + + desc3D.srcXInBytes = desc2D.srcXInBytes; + desc3D.srcY = desc2D.srcY; + desc3D.srcZ = 0; + desc3D.srcLOD = 0; + desc3D.srcMemoryType = desc2D.srcMemoryType; + desc3D.srcHost = desc2D.srcHost; + desc3D.srcDevice = desc2D.srcDevice; + desc3D.srcArray = desc2D.srcArray; + desc3D.srcPitch = desc2D.srcPitch; + desc3D.srcHeight = 0; + + desc3D.dstXInBytes = desc2D.dstXInBytes; + desc3D.dstY = desc2D.dstY; + desc3D.dstZ = 0; + desc3D.dstLOD = 0; + desc3D.dstMemoryType = desc2D.dstMemoryType; + desc3D.dstHost = desc2D.dstHost; + desc3D.dstDevice = desc2D.dstDevice; + desc3D.dstArray = desc2D.dstArray; + desc3D.dstPitch = desc2D.dstPitch; + desc3D.dstHeight = 0; + + desc3D.WidthInBytes = desc2D.WidthInBytes; + desc3D.Height = desc2D.Height; + desc3D.Depth = 0; + + return desc3D; +} +}; \ No newline at end of file diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 4b36a70b6e..63dd2c712e 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -20,24 +20,11 @@ #include #include "hip_internal.hpp" +#include "hip_conversions.hpp" #include "platform/context.hpp" #include "platform/command.hpp" #include "platform/memory.hpp" -extern void getChannelOrderAndType(const hipChannelFormatDesc& desc, - enum hipTextureReadMode readMode, - cl_channel_order* channelOrder, - cl_channel_type* channelType); - -extern void getDrvChannelOrderAndType(const enum hipArray_Format Format, - unsigned int NumChannels, - cl_channel_order* channelOrder, - cl_channel_type* channelType); - -extern void setDescFromChannelType(cl_channel_type channelType, hipChannelFormatDesc* desc); - -extern void getByteSizeFromChannelFormatKind(enum hipChannelFormatKind channelFormatKind, size_t* byteSize); - amd::Memory* getMemoryObject(const void* ptr, size_t& offset) { amd::Memory *memObj = amd::MemObjMap::FindMemObj(ptr); if (memObj != nullptr) { @@ -282,14 +269,27 @@ hipError_t hipHostFree(void* ptr) { HIP_RETURN(hipErrorInvalidValue); } +hipError_t ihipArrayDestroy(hipArray* array) { + if (array == nullptr) { + return hipErrorInvalidValue; + } + + cl_mem memObj = reinterpret_cast(array->data); + if (is_valid(memObj) == false) { + return hipErrorInvalidValue; + } + + as_amd(memObj)->release(); + + delete array; + + return hipSuccess; +} + hipError_t hipFreeArray(hipArray* array) { HIP_INIT_API(hipFreeArray, array); - if (amd::SvmBuffer::malloced(array->data)) { - amd::SvmBuffer::free(*hip::getCurrentDevice()->asContext(), array->data); - HIP_RETURN(hipSuccess); - } - HIP_RETURN(hipErrorInvalidValue); + HIP_RETURN(ihipArrayDestroy(array)); } hipError_t hipMemGetAddressRange(hipDeviceptr_t* pbase, size_t* psize, hipDeviceptr_t dptr) { @@ -393,142 +393,228 @@ hipError_t hipMalloc3D(hipPitchedPtr* pitchedDevPtr, hipExtent extent) { HIP_RETURN(status); } -hipError_t hipArrayCreate(hipArray** array, const HIP_ARRAY_DESCRIPTOR* pAllocateArray) { +amd::Image* ihipImageCreate(const cl_channel_order channelOrder, + const cl_channel_type channelType, + const cl_mem_object_type imageType, + const size_t imageWidth, + const size_t imageHeight, + const size_t imageDepth, + const size_t imageArraySize, + const size_t imageRowPitch, + const size_t imageSlicePitch, + const uint32_t numMipLevels, + amd::Memory* buffer) { + const amd::Image::Format imageFormat({channelOrder, channelType}); + if (!imageFormat.isValid()) { + return nullptr; + } + + amd::Context& context = *hip::getCurrentDevice()->asContext(); + if (!imageFormat.isSupported(context, imageType)) { + return nullptr; + } + + const std::vector& devices = context.devices(); + if (!devices[0]->info().imageSupport_) { + return nullptr; + } + + if (!amd::Image::validateDimensions(devices, + imageType, + imageWidth, + imageHeight, + imageDepth, + imageArraySize)) { + return nullptr; + } + + // TODO validate the image descriptor. + + amd::Image* image = nullptr; + if (buffer != nullptr) { + switch (imageType) { + case CL_MEM_OBJECT_IMAGE1D: + case CL_MEM_OBJECT_IMAGE2D: + image = new (context) amd::Image(*buffer->asBuffer(), + imageType, + CL_MEM_READ_WRITE, + imageFormat, + imageWidth, + (imageHeight == 0) ? 1 : imageHeight, + (imageDepth == 0) ? 1 : imageDepth, + imageRowPitch, + imageSlicePitch); + break; + default: + ShouldNotReachHere(); + } + } else { + switch (imageType) { + case CL_MEM_OBJECT_IMAGE1D: + case CL_MEM_OBJECT_IMAGE2D: + case CL_MEM_OBJECT_IMAGE3D: + image = new (context) amd::Image(context, + imageType, + CL_MEM_READ_WRITE, + imageFormat, + imageWidth, + (imageHeight == 0) ? 1 : imageHeight, + (imageDepth == 0) ? 1 : imageDepth, + imageWidth * imageFormat.getElementSize(), /* row pitch */ + imageWidth * imageHeight * imageFormat.getElementSize(), /* slice pitch */ + numMipLevels); + break; + case CL_MEM_OBJECT_IMAGE1D_ARRAY: + image = new (context) amd::Image(context, + imageType, + CL_MEM_READ_WRITE, + imageFormat, + imageWidth, + imageArraySize, + 1, /* image depth */ + imageWidth * imageFormat.getElementSize(), + imageWidth * imageHeight * imageFormat.getElementSize(), + numMipLevels); + break; + case CL_MEM_OBJECT_IMAGE2D_ARRAY: + image = new (context) amd::Image(context, + imageType, + CL_MEM_READ_WRITE, + imageFormat, + imageWidth, + imageHeight, + imageArraySize, + imageWidth * imageFormat.getElementSize(), + imageWidth * imageHeight * imageFormat.getElementSize(), + numMipLevels); + break; + default: + ShouldNotReachHere(); + } + } + + if (image == nullptr) { + return nullptr; + } + + if (!image->create(nullptr)) { + delete image; + return nullptr; + } + + return image; +} + +hipError_t ihipArrayCreate(hipArray** array, + const HIP_ARRAY3D_DESCRIPTOR* pAllocateArray, + unsigned int numMipmapLevels) { + // NumChannels specifies the number of packed components per HIP array element; it may be 1, 2, or 4; + if ((pAllocateArray->NumChannels != 1) && + (pAllocateArray->NumChannels != 2) && + (pAllocateArray->NumChannels != 4)) { + return hipErrorInvalidValue; + } + + if ((pAllocateArray->Flags & hipArraySurfaceLoadStore) || + (pAllocateArray->Flags & hipArrayCubemap) || + (pAllocateArray->Flags & hipArrayTextureGather)) { + return hipErrorNotSupported; + } + + const cl_channel_order channelOrder = hip::getCLChannelOrder(pAllocateArray->NumChannels); + const cl_channel_type channelType = hip::getCLChannelType(pAllocateArray->Format, hipReadModeElementType); + const cl_mem_object_type imageType = hip::getCLMemObjectType(pAllocateArray->Width, + pAllocateArray->Height, + pAllocateArray->Depth, + pAllocateArray->Flags); + + amd::Image* image = ihipImageCreate(channelOrder, + channelType, + imageType, + pAllocateArray->Width, + pAllocateArray->Height, + pAllocateArray->Depth, + // The number of layers is determined by the depth extent. + pAllocateArray->Depth, /* array size */ + 0, /* row pitch */ + 0, /* slice pitch */ + numMipmapLevels, + nullptr /* buffer */); + + if (image == nullptr) { + return hipErrorInvalidValue; + } + + cl_mem memObj = as_cl(image); + *array = new hipArray{reinterpret_cast(memObj)}; + + // It is UB to call hipGet*() on an array created via hipArrayCreate()/hipArray3DCreate(). + // This is due to hip not differentiating between runtime and driver types. + // TODO change the hipArray struct in driver_types.h. + (*array)->desc = hip::getChannelFormatDesc(pAllocateArray->NumChannels, pAllocateArray->Format); + (*array)->width = pAllocateArray->Width; + (*array)->height = pAllocateArray->Height; + (*array)->depth = pAllocateArray->Depth; + (*array)->Format = pAllocateArray->Format; + (*array)->NumChannels = pAllocateArray->NumChannels; + + return hipSuccess; +} + +hipError_t hipArrayCreate(hipArray** array, + const HIP_ARRAY_DESCRIPTOR* pAllocateArray) { HIP_INIT_API(hipArrayCreate, array, pAllocateArray); - if (pAllocateArray->Width == 0) { - HIP_RETURN(hipErrorInvalidValue); - } + HIP_ARRAY3D_DESCRIPTOR desc = {pAllocateArray->Width, + pAllocateArray->Height, + 0, /* Depth */ + pAllocateArray->Format, + pAllocateArray->NumChannels, + hipArrayDefault /* Flags */}; - *array = (hipArray*)malloc(sizeof(hipArray)); - array[0]->width = pAllocateArray->Width; - array[0]->height = pAllocateArray->Height; - array[0]->isDrv = true; - array[0]->textureType = hipTextureType2D; - void** ptr = &array[0]->data; - - cl_channel_order channelOrder; - cl_channel_type channelType; - getDrvChannelOrderAndType(pAllocateArray->Format, pAllocateArray->NumChannels, - &channelOrder, &channelType); - - const cl_image_format image_format = { channelOrder, channelType }; - setDescFromChannelType(channelType, &(array[0]->desc)); - - size_t pitch = 0; - hipError_t status = ihipMallocPitch(ptr, &pitch, array[0]->width, array[0]->height, 1, CL_MEM_OBJECT_IMAGE2D, - &image_format); - - HIP_RETURN(status); + HIP_RETURN(ihipArrayCreate(array, &desc, 0)); } -hipError_t hipMallocArray(hipArray** array, const hipChannelFormatDesc* desc, - size_t width, size_t height, unsigned int flags) { + +hipError_t hipMallocArray(hipArray** array, + const hipChannelFormatDesc* desc, + size_t width, + size_t height, + unsigned int flags) { HIP_INIT_API(hipMallocArray, array, desc, width, height, flags); - if (width == 0) { - HIP_RETURN(hipErrorInvalidValue); - } + HIP_ARRAY3D_DESCRIPTOR allocateArray = {width, + height, + 0, /* Depth */ + hip::getArrayFormat(*desc), + hip::getNumChannels(*desc), + flags}; - *array = (hipArray*)malloc(sizeof(hipArray)); - array[0]->type = flags; - array[0]->width = width; - array[0]->height = height; - array[0]->depth = 1; - array[0]->desc = *desc; - array[0]->isDrv = false; - array[0]->textureType = hipTextureType2D; - void** ptr = &array[0]->data; - - cl_channel_order channelOrder; - cl_channel_type channelType; - getChannelOrderAndType(*desc, hipReadModeElementType, &channelOrder, &channelType); - - const cl_image_format image_format = { channelOrder, channelType }; - - // Dummy flags check - switch (flags) { - case hipArrayLayered: - case hipArrayCubemap: - case hipArraySurfaceLoadStore: - case hipArrayTextureGather: - assert(0 && "Unspported"); - break; - case hipArrayDefault: - default: - break; - } - size_t pitch = 0; - hipError_t status = ihipMallocPitch(ptr, &pitch, width, height, 1, CL_MEM_OBJECT_IMAGE2D, - &image_format); - - HIP_RETURN(status); + HIP_RETURN(ihipArrayCreate(array, &allocateArray, 0 /* numMipLevels */)); } -hipError_t hipArray3DCreate(hipArray** array, const HIP_ARRAY3D_DESCRIPTOR* pAllocateArray) { +hipError_t hipArray3DCreate(hipArray** array, + const HIP_ARRAY3D_DESCRIPTOR* pAllocateArray) { HIP_INIT_API(hipArray3DCreate, array, pAllocateArray); - *array = (hipArray*)malloc(sizeof(hipArray)); - array[0]->type = pAllocateArray->Flags; - array[0]->width = pAllocateArray->Width; - array[0]->height = pAllocateArray->Height; - array[0]->depth = pAllocateArray->Depth; - array[0]->Format = pAllocateArray->Format; - array[0]->NumChannels = pAllocateArray->NumChannels; - array[0]->isDrv = true; - array[0]->textureType = hipTextureType3D; - void** ptr = &array[0]->data; - - cl_channel_order channelOrder; - cl_channel_type channelType; - getDrvChannelOrderAndType(pAllocateArray->Format, pAllocateArray->NumChannels, - &channelOrder, &channelType); - - const cl_image_format image_format = { channelOrder, channelType }; - size_t pitch = 0; - hipError_t status = ihipMallocPitch(ptr, &pitch, array[0]->width, array[0]->height, array[0]->depth, CL_MEM_OBJECT_IMAGE3D, - &image_format); - - HIP_RETURN(status); + HIP_RETURN(ihipArrayCreate(array, pAllocateArray, 0 /* numMipLevels */)); } -hipError_t hipMalloc3DArray(hipArray_t* array, const struct hipChannelFormatDesc* desc, - struct hipExtent extent, unsigned int flags) { +hipError_t hipMalloc3DArray(hipArray_t* array, + const hipChannelFormatDesc* desc, + hipExtent extent, + unsigned int flags) { + // TODO overload operator<<(ostream&, hipExtent&). HIP_INIT_API(hipMalloc3DArray, array, desc, &extent, flags); - *array = (hipArray*)malloc(sizeof(hipArray)); - array[0]->type = flags; - array[0]->width = extent.width; - array[0]->height = extent.height; - array[0]->depth = extent.depth; - array[0]->desc = *desc; - array[0]->isDrv = false; - array[0]->textureType = hipTextureType3D; - void** ptr = &array[0]->data; + HIP_ARRAY3D_DESCRIPTOR allocateArray = {extent.width, + extent.height, + extent.depth, + hip::getArrayFormat(*desc), + hip::getNumChannels(*desc), + flags}; - cl_channel_order channelOrder; - cl_channel_type channelType; - getChannelOrderAndType(*desc, hipReadModeElementType, &channelOrder, &channelType); - - const cl_image_format image_format = { channelOrder, channelType }; - - // Dummy flags check - switch (flags) { - case hipArrayCubemap: - case hipArraySurfaceLoadStore: - case hipArrayTextureGather: - assert(0 && "Unspported"); - break; - case hipArrayLayered: - case hipArrayDefault: - default: - break; - } - size_t pitch = 0; - hipError_t status = ihipMallocPitch(ptr, &pitch, extent.width, extent.height, extent.depth, - CL_MEM_OBJECT_IMAGE3D, &image_format); - - HIP_RETURN(status); + HIP_RETURN(ihipArrayCreate(array, &allocateArray, 0)); } hipError_t hipHostGetFlags(unsigned int* flagsPtr, void* hostPtr) { @@ -708,40 +794,28 @@ hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* symbolName, size_t co HIP_RETURN(hipMemcpyAsync(dst, device_ptr, count, kind, stream)); } -hipError_t hipMemcpyHtoD(hipDeviceptr_t dst, void* src, size_t sizeBytes) { - HIP_INIT_API(hipMemcpyHtoD, dst, src, sizeBytes); +hipError_t hipMemcpyHtoD(hipDeviceptr_t dstDevice, + void* srcHost, + size_t ByteCount) { + HIP_INIT_API(hipMemcpyHtoD, dstDevice, srcHost, ByteCount); - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - HIP_RETURN(ihipMemcpy(reinterpret_cast(dst), (const void*) src, sizeBytes, hipMemcpyHostToDevice, *queue)); + HIP_RETURN(ihipMemcpy(dstDevice, srcHost, ByteCount, hipMemcpyHostToDevice, *hip::getQueue(nullptr))); } -hipError_t hipMemcpyDtoH(void* dst, hipDeviceptr_t src, size_t sizeBytes) { - HIP_INIT_API(hipMemcpyDtoH, dst, src, sizeBytes); +hipError_t hipMemcpyDtoH(void* dstHost, + hipDeviceptr_t srcDevice, + size_t ByteCount) { + HIP_INIT_API(hipMemcpyDtoH, dstHost, srcDevice, ByteCount); - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - HIP_RETURN(ihipMemcpy(reinterpret_cast(dst), (const void*) src, sizeBytes, hipMemcpyDeviceToHost, *queue)); + HIP_RETURN(ihipMemcpy(dstHost, srcDevice, ByteCount, hipMemcpyDeviceToHost, *hip::getQueue(nullptr))); } -hipError_t hipMemcpyDtoD(hipDeviceptr_t dst, hipDeviceptr_t src, size_t sizeBytes) { - HIP_INIT_API(hipMemcpyDtoD, dst, src, sizeBytes); +hipError_t hipMemcpyDtoD(hipDeviceptr_t dstDevice, + hipDeviceptr_t srcDevice, + size_t ByteCount) { + HIP_INIT_API(hipMemcpyDtoD, dstDevice, srcDevice, ByteCount); - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - HIP_RETURN(ihipMemcpy(reinterpret_cast(dst), (const void*) src, sizeBytes, hipMemcpyDeviceToDevice, *queue)); -} - -hipError_t hipMemcpyHtoH(void* dst, void* src, size_t sizeBytes) { - HIP_INIT_API(NONE, dst, src, sizeBytes); - - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - HIP_RETURN(ihipMemcpy(reinterpret_cast(dst), (const void*) src, sizeBytes, hipMemcpyHostToHost, *queue)); + HIP_RETURN(ihipMemcpy(dstDevice, srcDevice, ByteCount, hipMemcpyDeviceToDevice, *hip::getQueue(nullptr))); } hipError_t hipMemcpyAsync(void* dst, const void* src, size_t sizeBytes, @@ -753,90 +827,69 @@ hipError_t hipMemcpyAsync(void* dst, const void* src, size_t sizeBytes, HIP_RETURN(ihipMemcpy(dst, src, sizeBytes, kind, *queue, true)); } - -hipError_t hipMemcpyHtoDAsync(hipDeviceptr_t dst, void* src, size_t sizeBytes, +hipError_t hipMemcpyHtoDAsync(hipDeviceptr_t dstDevice, + void* srcHost, + size_t ByteCount, hipStream_t stream) { - HIP_INIT_API(hipMemcpyHtoDAsync, dst, src, sizeBytes, stream); + HIP_INIT_API(hipMemcpyHtoDAsync, dstDevice, srcHost, ByteCount, stream); - amd::HostQueue* queue = hip::getQueue(stream); - - HIP_RETURN(ihipMemcpy(reinterpret_cast(dst), (const void*) src, sizeBytes, hipMemcpyHostToDevice, - *queue, true)); + HIP_RETURN(ihipMemcpy(dstDevice, srcHost, ByteCount, hipMemcpyHostToDevice, *hip::getQueue(stream), true)); } -hipError_t hipMemcpyDtoDAsync(hipDeviceptr_t dst, hipDeviceptr_t src, size_t sizeBytes, +hipError_t hipMemcpyDtoDAsync(hipDeviceptr_t dstDevice, + hipDeviceptr_t srcDevice, + size_t ByteCount, hipStream_t stream) { - HIP_INIT_API(hipMemcpyDtoDAsync, dst, src, sizeBytes, stream); + HIP_INIT_API(hipMemcpyDtoDAsync, dstDevice, srcDevice, ByteCount, stream); - amd::HostQueue* queue = hip::getQueue(stream); - - HIP_RETURN(ihipMemcpy(reinterpret_cast(dst), (const void*) src, sizeBytes, hipMemcpyDeviceToDevice, - *queue, true)); + HIP_RETURN(ihipMemcpy(dstDevice, srcDevice, ByteCount, hipMemcpyDeviceToDevice, *hip::getQueue(stream), true)); } -hipError_t hipMemcpyDtoHAsync(void* dst, hipDeviceptr_t src, size_t sizeBytes, +hipError_t hipMemcpyDtoHAsync(void* dstHost, + hipDeviceptr_t srcDevice, + size_t ByteCount, hipStream_t stream) { - HIP_INIT_API(hipMemcpyDtoHAsync, dst, src, sizeBytes, stream); + HIP_INIT_API(hipMemcpyDtoHAsync, dstHost, srcDevice, ByteCount, stream); - amd::HostQueue* queue = hip::getQueue(stream); - - HIP_RETURN(ihipMemcpy(reinterpret_cast(dst), (const void*) src, sizeBytes, hipMemcpyDeviceToHost, - *queue, true)); + HIP_RETURN(ihipMemcpy(dstHost, srcDevice, ByteCount, hipMemcpyDeviceToHost, *hip::getQueue(stream), true)); } -hipError_t ihipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, - size_t height, hipMemcpyKind kind, amd::HostQueue& queue, - bool isAsync = false) { - // Create buffer rectangle info structure - amd::BufferRect srcRect; - amd::BufferRect dstRect; +hipError_t ihipMemcpyAtoD(hipArray* srcArray, + void* dstDevice, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D region, + size_t dstRowPitch, + size_t dstSlicePitch, + hipStream_t stream, + bool isAsync = false) { + // TODO VDI doesn't support 2D/3D image to buffer copy. + (void)dstRowPitch; + (void)dstSlicePitch; - size_t region[3] = {width, height, 1}; - size_t src_slice_pitch = spitch * height; - size_t dst_slice_pitch = dpitch * height; - size_t sOrigin[3] = { }; - size_t dOrigin[3] = { }; - amd::Memory* srcMemory = getMemoryObject(src, sOrigin[0]); - amd::Memory* dstMemory = getMemoryObject(dst, dOrigin[0]); - - if (src_slice_pitch == 0 || - dst_slice_pitch == 0 || - dst == nullptr || - src == nullptr) { - return hipSuccess; - } - - if (!srcRect.create(sOrigin, region, spitch, src_slice_pitch) || - !dstRect.create(dOrigin, region, dpitch, dst_slice_pitch)) { + cl_mem srcMemObj = reinterpret_cast(srcArray->data); + if (is_valid(srcMemObj) == false) { return hipErrorInvalidValue; } - amd::Command* command = nullptr; - amd::Command::EventWaitList waitList; + amd::Image* srcImage = as_amd(srcMemObj)->asImage(); + size_t offset = 0; + amd::Memory* dstMemory = getMemoryObject(dstDevice, offset); + assert(offset != 0); - amd::Coord3D srcStart(srcRect.start_, 0, 0); - amd::Coord3D dstStart(dstRect.start_, 0, 0); - amd::Coord3D size(region[0], region[1], region[2]); - - if (((srcMemory == nullptr) && (dstMemory == nullptr)) || - (kind == hipMemcpyHostToHost)) { - for(unsigned int y = 0; y < height; y++) { - void* pDst = reinterpret_cast(reinterpret_cast(dst) + y * dpitch); - void* pSrc = reinterpret_cast(reinterpret_cast(src) + y * spitch); - memcpy(pDst, pSrc, width); - } - return hipSuccess; - } else if ((srcMemory == nullptr) && (dstMemory != nullptr)) { - command = new amd::WriteMemoryCommand(queue, CL_COMMAND_WRITE_BUFFER_RECT, waitList, - *dstMemory->asBuffer(), dstStart, size, src, dstRect, srcRect); - } else if ((srcMemory != nullptr) && (dstMemory == nullptr)) { - command = new amd::ReadMemoryCommand(queue, CL_COMMAND_READ_BUFFER_RECT, waitList, - *srcMemory->asBuffer(), srcStart, size, dst, srcRect, dstRect); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - command = new amd::CopyMemoryCommand(queue, CL_COMMAND_COPY_BUFFER_RECT, waitList, *srcMemory->asBuffer(), - *dstMemory->asBuffer(), srcStart, dstStart, size, srcRect, dstRect); + if (!srcImage->validateRegion(srcOrigin, region)) { + return hipErrorInvalidValue; } + amd::CopyMemoryCommand* command = new amd::CopyMemoryCommand(*hip::getQueue(stream), + CL_COMMAND_COPY_IMAGE_TO_BUFFER, + amd::Command::EventWaitList{}, + *srcImage, + *dstMemory, + srcOrigin, + dstOrigin, + region); + if (command == nullptr) { return hipErrorOutOfMemory; } @@ -850,578 +903,686 @@ hipError_t ihipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch return hipSuccess; } +hipError_t ihipMemcpyDtoA(void* srcDevice, + hipArray* dstArray, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D region, + size_t srcRowPitch, + size_t srcSlicePitch, + hipStream_t stream, + bool isAsync = false) { + // TODO VDI doesn't support 2D/3D buffer to image copy. + (void)srcRowPitch; + (void)srcSlicePitch; + + cl_mem dstMemObj = reinterpret_cast(dstArray->data); + if (is_valid(dstMemObj) == false) { + return hipErrorInvalidValue; + } + + size_t offset = 0; + amd::Memory* srcMemory = getMemoryObject(srcDevice, offset); + assert(offset != 0); + amd::Image* dstImage = as_amd(dstMemObj)->asImage(); + + const size_t copySizeInBytes = region[0] * region[1] * region[2] * dstImage->getImageFormat().getElementSize(); + if (!srcMemory->validateRegion(srcOrigin, {copySizeInBytes, 0, 0}) || + !dstImage->validateRegion(dstOrigin, region)) { + return hipErrorInvalidValue; + } + + amd::CopyMemoryCommand* command = new amd::CopyMemoryCommand(*hip::getQueue(stream), + CL_COMMAND_COPY_BUFFER_TO_IMAGE, + amd::Command::EventWaitList{}, + *srcMemory, + *dstImage, + srcOrigin, + dstOrigin, + region); + + if (command == nullptr) { + return hipErrorOutOfMemory; + } + + command->enqueue(); + if (!isAsync) { + command->awaitCompletion(); + } + command->release(); + + return hipSuccess; +} + +hipError_t ihipMemcpyDtoD(void* srcDevice, + void* dstDevice, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D copyRegion, + size_t srcRowPitch, + size_t srcSlicePitch, + size_t dstRowPitch, + size_t dstSlicePitch, + hipStream_t stream, + bool isAsync = false) { + size_t srcOffset = 0; + amd::Memory *srcMemory = getMemoryObject(srcDevice, srcOffset); + size_t dstOffset = 0; + amd::Memory *dstMemory = getMemoryObject(dstDevice, dstOffset); + + amd::BufferRect srcRect; + if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::Coord3D srcStart(srcRect.start_ + srcOffset, 0, 0); + amd::Coord3D srcEnd(srcRect.end_ + srcOffset, 1, 1); + if (!srcMemory->validateRegion(srcStart, srcEnd)) { + return hipErrorInvalidValue; + } + + amd::BufferRect dstRect; + if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), dstRowPitch, dstSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::Coord3D dstStart(dstRect.start_ + dstOffset, 0, 0); + amd::Coord3D dstEnd(dstRect.end_ + dstOffset, 1, 1); + if (!dstMemory->validateRegion(dstStart, dstEnd)) { + return hipErrorInvalidValue; + } + + amd::CopyMemoryCommand* command = new amd::CopyMemoryCommand(*hip::getQueue(stream), + CL_COMMAND_COPY_BUFFER_RECT, + amd::Command::EventWaitList{}, + *srcMemory, + *dstMemory, + srcStart, + dstStart, + copyRegion, + srcRect, + dstRect); + + if (command == nullptr) { + return hipErrorOutOfMemory; + } + + command->enqueue(); + if (!isAsync) { + command->awaitCompletion(); + } + command->release(); + + return hipSuccess; +} + +hipError_t ihipMemcpyDtoH(void* srcDevice, + void* dstHost, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D copyRegion, + size_t srcRowPitch, + size_t srcSlicePitch, + size_t dstRowPitch, + size_t dstSlicePitch, + hipStream_t stream, + bool isAsync = false) { + size_t srcOffset = 0; + amd::Memory *srcMemory = getMemoryObject(srcDevice, srcOffset); + + amd::BufferRect srcRect; + if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::Coord3D srcStart(srcRect.start_ + srcOffset, 0, 0); + amd::Coord3D srcEnd(srcRect.end_ + srcOffset, 1, 1); + if (!srcMemory->validateRegion(srcStart, srcEnd)) { + return hipErrorInvalidValue; + } + + amd::BufferRect dstRect; + if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), dstRowPitch, dstSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::ReadMemoryCommand* command = new amd::ReadMemoryCommand(*hip::getQueue(stream), + CL_COMMAND_READ_BUFFER_RECT, + amd::Command::EventWaitList{}, + *srcMemory, + srcStart, + copyRegion, + dstHost, + srcRect, + dstRect); + + if (command == nullptr) { + return hipErrorOutOfMemory; + } + + command->enqueue(); + if (!isAsync) { + command->awaitCompletion(); + } + command->release(); + + return hipSuccess; +} + +hipError_t ihipMemcpyHtoD(const void* srcHost, + void* dstDevice, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D copyRegion, + size_t srcRowPitch, + size_t srcSlicePitch, + size_t dstRowPitch, + size_t dstSlicePitch, + hipStream_t stream, + bool isAsync = false) { + size_t dstOffset = 0; + amd::Memory *dstMemory = getMemoryObject(dstDevice, dstOffset); + + amd::BufferRect srcRect; + if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::BufferRect dstRect; + if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), dstRowPitch, dstSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::Coord3D dstStart(dstRect.start_ + dstOffset, 0, 0); + amd::Coord3D dstEnd(dstRect.end_ + dstOffset, 1, 1); + if (!dstMemory->validateRegion(dstStart, dstEnd)) { + return hipErrorInvalidValue; + } + + amd::WriteMemoryCommand* command = new amd::WriteMemoryCommand(*hip::getQueue(stream), + CL_COMMAND_WRITE_BUFFER_RECT, + amd::Command::EventWaitList{}, + *dstMemory, + dstStart, + copyRegion, + srcHost, + dstRect, + srcRect); + + if (command == nullptr) { + return hipErrorOutOfMemory; + } + + command->enqueue(); + if (!isAsync) { + command->awaitCompletion(); + } + command->release(); + + return hipSuccess; +} + +hipError_t ihipMemcpyHtoH(const void* srcHost, + void* dstHost, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D copyRegion, + size_t srcRowPitch, + size_t srcSlicePitch, + size_t dstRowPitch, + size_t dstSlicePitch) { + amd::BufferRect srcRect; + if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::BufferRect dstRect; + if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), dstRowPitch, dstSlicePitch)) { + return hipErrorInvalidValue; + } + + for (size_t slice = 0; slice < copyRegion[2]; slice++) { + for (size_t row = 0; row < copyRegion[1]; row++) { + const void* srcRow = static_cast(srcHost) + srcRect.start_ + row * srcRect.rowPitch_ + slice * srcRect.slicePitch_; + void* dstRow = static_cast(dstHost) + dstRect.start_ + row * dstRect.rowPitch_ + slice * dstRect.slicePitch_; + std::memcpy(dstRow, srcRow, copyRegion[0]); + } + } + + return hipSuccess; +} + +hipError_t ihipMemcpyAtoA(hipArray* srcArray, + hipArray* dstArray, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D region, + hipStream_t stream, + bool isAsync = false) { + cl_mem srcMemObj = reinterpret_cast(srcArray->data); + cl_mem dstMemObj = reinterpret_cast(dstArray->data); + if (!is_valid(srcMemObj) || !is_valid(dstMemObj)) { + return hipErrorInvalidValue; + } + + amd::Image* srcImage = as_amd(srcMemObj)->asImage(); + amd::Image* dstImage = as_amd(dstMemObj)->asImage(); + + if (!srcImage->validateRegion(srcOrigin, region) || + !dstImage->validateRegion(dstOrigin, region)) { + return hipErrorInvalidValue; + } + + amd::CopyMemoryCommand* command = new amd::CopyMemoryCommand(*hip::getQueue(stream), + CL_COMMAND_COPY_IMAGE, + amd::Command::EventWaitList{}, + *srcImage, + *dstImage, + srcOrigin, + dstOrigin, + region); + + if (command == nullptr) { + return hipErrorOutOfMemory; + } + + command->enqueue(); + if (!isAsync) { + command->awaitCompletion(); + } + command->release(); + + return hipSuccess; +} + +hipError_t ihipMemcpyHtoA(const void* srcHost, + hipArray* dstArray, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D copyRegion, + size_t srcRowPitch, + size_t srcSlicePitch, + hipStream_t stream, + bool isAsync = false) { + if (srcHost == nullptr) { + return hipErrorInvalidValue; + } + + cl_mem dstMemObj = reinterpret_cast(dstArray->data); + if (is_valid(dstMemObj) == false) { + return hipErrorInvalidValue; + } + + amd::BufferRect srcRect; + if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::Image* dstImage = as_amd(dstMemObj)->asImage(); + // HIP assumes the width is in bytes, but OCL assumes it's in pixels. + const size_t elementSize = dstImage->getImageFormat().getElementSize(); + static_cast(dstOrigin)[0] /= elementSize; + static_cast(copyRegion)[0] /= elementSize; + + if (!dstImage->validateRegion(dstOrigin, copyRegion)) { + return hipErrorInvalidValue; + } + + amd::WriteMemoryCommand* command = new amd::WriteMemoryCommand(*hip::getQueue(stream), + CL_COMMAND_WRITE_IMAGE, + amd::Command::EventWaitList{}, + *dstImage, + dstOrigin, + copyRegion, + static_cast(srcHost) + srcRect.start_, + srcRowPitch, + srcSlicePitch); + + if (command == nullptr) { + return hipErrorOutOfMemory; + } + + command->enqueue(); + if (!isAsync) { + command->awaitCompletion(); + } + command->release(); + + return hipSuccess; +} + +hipError_t ihipMemcpyAtoH(hipArray* srcArray, + void* dstHost, + amd::Coord3D srcOrigin, + amd::Coord3D dstOrigin, + amd::Coord3D copyRegion, + size_t dstRowPitch, + size_t dstSlicePitch, + hipStream_t stream, + bool isAsync = false) { + cl_mem srcMemObj = reinterpret_cast(srcArray->data); + if (!is_valid(srcMemObj)) { + return hipErrorInvalidValue; + } + + if (dstHost == nullptr) { + return hipErrorInvalidValue; + } + + amd::BufferRect dstRect; + if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), dstRowPitch, dstSlicePitch)) { + return hipErrorInvalidValue; + } + + + amd::Image* srcImage = as_amd(srcMemObj)->asImage(); + // HIP assumes the width is in bytes, but OCL assumes it's in pixels. + const size_t elementSize = srcImage->getImageFormat().getElementSize(); + static_cast(srcOrigin)[0] /= elementSize; + static_cast(copyRegion)[0] /= elementSize; + + if (!srcImage->validateRegion(srcOrigin, copyRegion) || + !srcImage->isRowSliceValid(dstRowPitch, dstSlicePitch, copyRegion[0], copyRegion[1])) { + return hipErrorInvalidValue; + } + + amd::ReadMemoryCommand* command = new amd::ReadMemoryCommand(*hip::getQueue(stream), + CL_COMMAND_READ_IMAGE, + amd::Command::EventWaitList{}, + *srcImage, + srcOrigin, + copyRegion, + static_cast(dstHost) + dstRect.start_, + dstRowPitch, + dstSlicePitch); + + if (command == nullptr) { + return hipErrorOutOfMemory; + } + + command->enqueue(); + if (!isAsync) { + command->awaitCompletion(); + } + command->release(); + + return hipSuccess; +} + +hipError_t ihipMemcpyParam3D(const hipMemcpy3DParms* pCopy, + hipStream_t stream, + bool isAsync = false) { + // If {src/dst}MemoryType is hipMemoryTypeUnified, {src/dst}Device and {src/dst}Pitch specify the (unified virtual address space) + // base address of the source data and the bytes per row to apply. {src/dst}Array is ignored. + hipMemoryType srcMemoryType = pCopy->srcMemoryType; + if (srcMemoryType == hipMemoryTypeUnified) { + srcMemoryType = amd::MemObjMap::FindMemObj(pCopy->srcDevice) ? hipMemoryTypeDevice : hipMemoryTypeHost; + } + hipMemoryType dstMemoryType = pCopy->dstMemoryType; + if (dstMemoryType == hipMemoryTypeUnified) { + dstMemoryType = amd::MemObjMap::FindMemObj(pCopy->dstDevice) ? hipMemoryTypeDevice : hipMemoryTypeHost; + } + + amd::Coord3D srcOrigin = {pCopy->srcXInBytes, pCopy->srcY, pCopy->srcZ}; + amd::Coord3D dstOrigin = {pCopy->dstXInBytes, pCopy->dstY, pCopy->dstZ}; + amd::Coord3D copyRegion = {pCopy->WidthInBytes, (pCopy->Height != 0) ? pCopy->Height : 1, (pCopy->Depth != 0) ? pCopy->Depth : 1}; + + if ((srcMemoryType == hipMemoryTypeHost) && (dstMemoryType == hipMemoryTypeHost)) { + // Host to Host. + return ihipMemcpyHtoH(pCopy->srcHost, pCopy->dstHost, srcOrigin, dstOrigin, copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight); + } else if ((srcMemoryType == hipMemoryTypeHost) && (dstMemoryType == hipMemoryTypeDevice)) { + // Host to Device. + return ihipMemcpyHtoD(pCopy->srcHost, pCopy->dstDevice, srcOrigin, dstOrigin, copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, stream, isAsync); + } else if ((srcMemoryType == hipMemoryTypeDevice) && (dstMemoryType == hipMemoryTypeHost)) { + // Device to Host. + return ihipMemcpyDtoH(pCopy->srcDevice, pCopy->dstHost, srcOrigin, dstOrigin, copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, stream, isAsync); + } else if ((srcMemoryType == hipMemoryTypeDevice) && (dstMemoryType == hipMemoryTypeDevice)) { + // Device to Device. + return ihipMemcpyDtoD(pCopy->srcDevice, pCopy->dstDevice, srcOrigin, dstOrigin, copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, stream, isAsync); + } else if ((srcMemoryType == hipMemoryTypeHost) && (dstMemoryType == hipMemoryTypeArray)) { + // Host to Image. + return ihipMemcpyHtoA(pCopy->srcHost, pCopy->dstArray, srcOrigin, dstOrigin, copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, stream, isAsync); + } else if ((srcMemoryType == hipMemoryTypeArray) && (dstMemoryType == hipMemoryTypeHost)) { + // Image to Host. + return ihipMemcpyAtoH(pCopy->srcArray, pCopy->dstHost, srcOrigin, dstOrigin, copyRegion, pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, stream, isAsync); + } else if ((srcMemoryType == hipMemoryTypeDevice) && (dstMemoryType == hipMemoryTypeArray)) { + // Device to Image. + return ihipMemcpyDtoA(pCopy->srcDevice, pCopy->dstArray, srcOrigin, dstOrigin, copyRegion, pCopy->srcPitch, pCopy->srcPitch * pCopy->srcHeight, stream, isAsync); + } else if ((srcMemoryType == hipMemoryTypeArray) && (dstMemoryType == hipMemoryTypeDevice)) { + // Image to Device. + return ihipMemcpyAtoD(pCopy->srcArray, pCopy->dstDevice, srcOrigin, dstOrigin, copyRegion, pCopy->dstPitch, pCopy->dstPitch * pCopy->dstHeight, stream, isAsync); + } else if ((srcMemoryType == hipMemoryTypeArray) && (dstMemoryType == hipMemoryTypeArray)) { + // Image to Image. + return ihipMemcpyAtoA(pCopy->srcArray, pCopy->dstArray, srcOrigin, dstOrigin, copyRegion, stream, isAsync); + } else { + ShouldNotReachHere(); + } + + return hipSuccess; +} + +hipError_t ihipMemcpyParam2D(const hip_Memcpy2D* pCopy, + hipStream_t stream, + bool isAsync = false) { + hipMemcpy3DParms desc = hip::getMemcpy3DParms(*pCopy); + + return ihipMemcpyParam3D(&desc, stream, isAsync); +} + +hipError_t ihipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, + size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { + hip_Memcpy2D desc = {}; + + desc.srcXInBytes = 0; + desc.srcY = 0; + desc.srcMemoryType = std::get<0>(hip::getMemoryType(kind)); + desc.srcHost = src; + desc.srcDevice = const_cast(src); + desc.srcArray = nullptr; // Ignored. + desc.srcPitch = spitch; + + desc.dstXInBytes = 0; + desc.dstY = 0; + desc.dstMemoryType = std::get<1>(hip::getMemoryType(kind)); + desc.dstHost = dst; + desc.dstDevice = dst; + desc.dstArray = nullptr; // Ignored. + desc.dstPitch = dpitch; + + desc.WidthInBytes = width; + desc.Height = height; + + return ihipMemcpyParam2D(&desc, stream, isAsync); +} + hipError_t hipMemcpyParam2D(const hip_Memcpy2D* pCopy) { HIP_INIT_API(hipMemcpyParam2D, pCopy); - hipError_t e = hipSuccess; - if (pCopy == nullptr) { - e = hipErrorInvalidValue; - } else { - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - e = ihipMemcpy2D(pCopy->dstArray->data, pCopy->WidthInBytes, pCopy->srcHost, pCopy->srcPitch, - pCopy->WidthInBytes, pCopy->Height, hipMemcpyDefault, *queue); - } - HIP_RETURN(e); + + HIP_RETURN(ihipMemcpyParam2D(pCopy, nullptr)); } hipError_t hipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, hipMemcpyKind kind) { HIP_INIT_API(hipMemcpy2D, dst, dpitch, src, spitch, width, height, kind); - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - HIP_RETURN(ihipMemcpy2D(dst, dpitch, src, spitch, width, height, kind, *queue)); + HIP_RETURN(ihipMemcpy2D(dst, dpitch, src, spitch, width, height, kind, nullptr)); } - hipError_t hipMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream) { HIP_INIT_API(hipMemcpy2DAsync, dst, dpitch, src, spitch, width, height, kind, stream); - amd::HostQueue* queue = hip::getQueue(stream); - - HIP_RETURN(ihipMemcpy2D(dst, dpitch, src, spitch, width, height, kind, *queue, true)); + HIP_RETURN(ihipMemcpy2D(dst, dpitch, src, spitch, width, height, kind, stream, true)); } -hipError_t hipMemcpy2DToArray(hipArray* dst, size_t wOffset, size_t hOffset, const void* src, - size_t spitch, size_t width, size_t height, hipMemcpyKind kind) { +hipError_t ihipMemcpy2DToArray(hipArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { + hip_Memcpy2D desc = {}; + + desc.srcXInBytes = 0; + desc.srcY = 0; + desc.srcMemoryType = std::get<0>(hip::getMemoryType(kind)); + desc.srcHost = const_cast(src); + desc.srcDevice = const_cast(src); + desc.srcArray = nullptr; + desc.srcPitch = spitch; + + desc.dstXInBytes = wOffset; + desc.dstY = hOffset; + desc.dstMemoryType = hipMemoryTypeArray; + desc.dstHost = nullptr; + desc.dstDevice = nullptr; + desc.dstArray = dst; + desc.dstPitch = 0; // Ignored. + + desc.WidthInBytes = width; + desc.Height = height; + + return ihipMemcpyParam2D(&desc, stream, isAsync); +} + +hipError_t hipMemcpy2DToArray(hipArray* dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, hipMemcpyKind kind) { HIP_INIT_API(hipMemcpy2DToArray, dst, wOffset, hOffset, src, spitch, width, height, kind); - if (dst->data == nullptr) { + HIP_RETURN(ihipMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind, nullptr)); +} + +hipError_t hipMemcpyToArray(hipArray* dst, size_t wOffset, size_t hOffset, const void* src, size_t count, hipMemcpyKind kind) { + HIP_INIT_API(hipMemcpyToArray, dst, wOffset, hOffset, src, count, kind); + + if (dst == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); + const size_t arrayHeight = (dst->height != 0) ? dst->height : 1; + const size_t witdthInBytes = count / arrayHeight; - size_t dpitch = dst->width; - getByteSizeFromChannelFormatKind(dst[0].desc.f, &dpitch); + const size_t height = (count / dst->width) / (hip::getElementSize(dst->Format) * dst->NumChannels); - if ((wOffset + width > (dpitch)) || width > spitch) { - HIP_RETURN(hipErrorInvalidDevicePointer); - } + HIP_RETURN(ihipMemcpy2DToArray(dst, wOffset, hOffset, src, 0 /* spitch */, witdthInBytes, height, kind, nullptr)); +} - // Create buffer rectangle info structure - amd::BufferRect srcRect; - amd::BufferRect dstRect; +hipError_t ihipMemcpy2DFromArray(void* dst, size_t dpitch, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { + hip_Memcpy2D desc = {}; - size_t region[3] = {width, height, 1}; - size_t src_slice_pitch = spitch * height; - size_t dst_slice_pitch = dpitch * height; - size_t sOrigin[3] = { }; - size_t dOrigin[3] = {wOffset, hOffset, 0}; - size_t offset = 0; - amd::Memory* srcMemory = getMemoryObject(src, offset); - amd::Memory* dstMemory = getMemoryObject(dst->data, offset); + desc.srcXInBytes = wOffsetSrc; + desc.srcY = hOffsetSrc; + desc.srcMemoryType = hipMemoryTypeArray; + desc.srcHost = nullptr; + desc.srcDevice = nullptr; + desc.srcArray = const_cast(src); + desc.srcPitch = 0; // Ignored. - assert(offset == 0); + desc.dstXInBytes = 0; + desc.dstY = 0; + desc.dstMemoryType = std::get<1>(hip::getMemoryType(kind)); + desc.dstHost = dst; + desc.dstDevice = dst; + desc.dstArray = nullptr; + desc.dstPitch = dpitch; - if (!srcRect.create(sOrigin, region, spitch, src_slice_pitch) || - !dstRect.create(dOrigin, region, dpitch, dst_slice_pitch)) { + desc.WidthInBytes = width; + desc.Height = height; + + return ihipMemcpyParam2D(&desc, stream, isAsync); +} + +hipError_t hipMemcpyFromArray(void* dst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffset, size_t count, hipMemcpyKind kind) { + HIP_INIT_API(hipMemcpyFromArray, dst, src, wOffsetSrc, hOffset, count, kind); + + if (src == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - amd::Command* command = nullptr; - amd::Command::EventWaitList waitList; + const size_t arrayHeight = (src->height != 0) ? src->height : 1; + const size_t witdthInBytes = count / arrayHeight; - amd::Coord3D srcStart(srcRect.start_, 0, 0); - amd::Coord3D dstStart(dstRect.start_, 0, 0); - amd::Coord3D size(region[0], region[1], region[2]); - - if (((srcMemory == nullptr) && (dstMemory == nullptr)) || - (kind == hipMemcpyHostToHost)) { - void* newDst = reinterpret_cast(reinterpret_cast(dst->data) - + dpitch * hOffset + wOffset); - for(unsigned int y = 0; y < height; y++) { - void* pDst = reinterpret_cast(reinterpret_cast(newDst) + y * dpitch); - void* pSrc = reinterpret_cast(reinterpret_cast(src) + y * spitch); - memcpy(pDst, pSrc, width); - } - HIP_RETURN(hipSuccess); - } else if ((srcMemory == nullptr) && (dstMemory != nullptr)) { - command = new amd::WriteMemoryCommand(*queue, CL_COMMAND_WRITE_BUFFER_RECT, waitList, - *dstMemory->asBuffer(), dstStart, size, src, dstRect, srcRect); - } else if ((srcMemory != nullptr) && (dstMemory == nullptr)) { - command = new amd::ReadMemoryCommand(*queue, CL_COMMAND_READ_BUFFER_RECT, waitList, - *srcMemory->asBuffer(), srcStart, size, dst, srcRect, dstRect); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_BUFFER_RECT, waitList, *srcMemory->asBuffer(), - *dstMemory->asBuffer(), srcStart, dstStart, size, srcRect, dstRect); - } - - if (command == nullptr) { - HIP_RETURN(hipErrorOutOfMemory); - } - - command->enqueue(); - command->awaitCompletion(); - command->release(); - - HIP_RETURN(hipSuccess); + const size_t height = (count / src->width) / (hip::getElementSize(src->Format) * src->NumChannels); + HIP_RETURN(ihipMemcpy2DFromArray(dst, 0 /* dpitch */, src, wOffsetSrc, hOffset, witdthInBytes, height, kind, nullptr)); } -hipError_t hipMemcpyToArray(hipArray* dstArray, size_t wOffset, size_t hOffset, const void* src, - size_t count, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyToArray, dstArray, wOffset, hOffset, src, count, kind); +hipError_t hipMemcpyHtoA(hipArray* dstArray, + size_t dstOffset, + const void* srcHost, + size_t ByteCount) { + HIP_INIT_API(hipMemcpyHtoA, dstArray, dstOffset, srcHost, ByteCount); - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - amd::Command* command = nullptr; - amd::Command::EventWaitList waitList; - - size_t sOffset = 0; - amd::Memory* srcMemory = getMemoryObject(src, sOffset); - size_t dOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dstArray->data, dOffset); - - assert(dOffset == 0); - - assert((kind == hipMemcpyHostToDevice) && "Invalid case"); - - if ((srcMemory == nullptr) && (dstMemory != nullptr)) { - command = new amd::WriteMemoryCommand(*queue, CL_COMMAND_WRITE_BUFFER, waitList, - *dstMemory->asBuffer(), {wOffset, hOffset}, count, src); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_BUFFER, waitList, - *srcMemory->asBuffer(),*dstMemory->asBuffer(), sOffset, {wOffset, hOffset}, count); - } else { - ShouldNotReachHere(); - } - - if (command == nullptr) { - HIP_RETURN(hipErrorOutOfMemory); - } - - command->enqueue(); - command->awaitCompletion(); - command->release(); - - HIP_RETURN(hipSuccess); + HIP_RETURN(ihipMemcpyHtoA(srcHost, dstArray, {0, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, 0, 0, nullptr)); } -hipError_t hipMemcpyFromArray(void* dst, hipArray_const_t srcArray, size_t wOffset, size_t hOffset, - size_t count, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyFromArray, dst, srcArray, wOffset, hOffset, count, kind); +hipError_t hipMemcpyAtoH(void* dstHost, + hipArray* srcArray, + size_t srcOffset, + size_t ByteCount) { + HIP_INIT_API(hipMemcpyAtoH, dstHost, srcArray, srcOffset, ByteCount); - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - amd::Command* command = nullptr; - amd::Command::EventWaitList waitList; - - size_t sOffset = 0; - amd::Memory* srcMemory = getMemoryObject(srcArray->data, sOffset); - size_t dOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dst, dOffset); - - assert(sOffset == 0); - - assert((kind == hipMemcpyDeviceToHost) && "Invalid case"); - - if ((srcMemory != nullptr) && (dstMemory == nullptr)) { - command = new amd::ReadMemoryCommand(*queue, CL_COMMAND_READ_BUFFER, waitList, - *srcMemory->asBuffer(), {wOffset, hOffset}, count, dst); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_BUFFER, waitList, - *srcMemory->asBuffer(), *dstMemory->asBuffer(), {wOffset, hOffset}, dOffset, count); - } else { - ShouldNotReachHere(); - } - - if (command == nullptr) { - HIP_RETURN(hipErrorOutOfMemory); - } - - command->enqueue(); - command->awaitCompletion(); - command->release(); - - HIP_RETURN(hipSuccess); + HIP_RETURN(ihipMemcpyAtoH(srcArray, dstHost, {srcOffset, 0, 0}, {0, 0, 0}, {ByteCount, 1, 1}, 0, 0, nullptr)); } -hipError_t hipMemcpyHtoA(hipArray* dstArray, size_t dstOffset, const void* srcHost, size_t count) { - HIP_INIT_API(hipMemcpyHtoA, dstArray, dstOffset, srcHost, count); - - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - amd::Command* command = nullptr; - amd::Command::EventWaitList waitList; - - size_t sOffset = 0; - amd::Memory* srcMemory = getMemoryObject(srcHost, sOffset); - size_t dOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dstArray->data, dOffset); - - assert(dOffset == 0); - - if ((srcMemory == nullptr) && (dstMemory != nullptr)) { - command = new amd::WriteMemoryCommand(*queue, CL_COMMAND_WRITE_BUFFER, waitList, - *dstMemory->asBuffer(), dstOffset, count, srcHost); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_BUFFER, waitList, - *srcMemory->asBuffer(), *dstMemory->asBuffer(), sOffset, dstOffset, count); - } else { - ShouldNotReachHere(); - } - - if (command == nullptr) { - HIP_RETURN(hipErrorOutOfMemory); - } - - command->enqueue(); - command->awaitCompletion(); - command->release(); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipMemcpyAtoH(void* dst, hipArray* srcArray, size_t srcOffset, size_t count) { - HIP_INIT_API(hipMemcpyAtoH, dst, srcArray, srcOffset, count); - - hip::syncStreams(); - amd::HostQueue* queue = hip::getNullStream(); - - amd::Command* command = nullptr; - amd::Command::EventWaitList waitList; - - size_t sOffset = 0; - amd::Memory* srcMemory = getMemoryObject(srcArray->data, sOffset); - size_t dOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dst, dOffset); - - assert(sOffset == 0); - - if ((srcMemory != nullptr) && (dstMemory == nullptr)) { - command = new amd::ReadMemoryCommand(*queue, CL_COMMAND_READ_BUFFER, waitList, - *srcMemory->asBuffer(), srcOffset, count, dst); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_BUFFER, waitList, - *srcMemory->asBuffer(), *dstMemory->asBuffer(), srcOffset, dOffset, count); - } else { - ShouldNotReachHere(); - } - - if (command == nullptr) { - HIP_RETURN(hipErrorOutOfMemory); - } - - command->enqueue(); - command->awaitCompletion(); - command->release(); - - HIP_RETURN(hipSuccess); -} - -hipError_t ihipMemcpy3D_V1(const struct hipMemcpy3DParms* p, hipStream_t stream, bool isAsync = false) { - const void* srcPtr = nullptr; - size_t srcElementSizeInBytes = sizeof(unsigned char); - size_t srcRowPitchInBytes = 0; - size_t srcSlicePitchInBytes = 0; - if (p->srcMemoryType == hipMemoryTypeHost) { - srcPtr = p->srcHost; - srcRowPitchInBytes = p->srcPitch; - srcSlicePitchInBytes = srcRowPitchInBytes * p->srcHeight; - } else if ((p->srcMemoryType == hipMemoryTypeDevice) || - (p->srcMemoryType == hipMemoryTypeUnified)) { - srcPtr = p->srcDevice; - srcRowPitchInBytes = p->srcPitch; - srcSlicePitchInBytes = srcRowPitchInBytes * p->srcHeight; - } else if (p->srcMemoryType == hipMemoryTypeArray) { - srcPtr = p->srcArray->data; - getByteSizeFromChannelFormatKind(p->srcArray->desc.f, &srcElementSizeInBytes); - srcElementSizeInBytes *= p->srcArray->NumChannels; - srcRowPitchInBytes = srcElementSizeInBytes * p->srcArray->width; - srcSlicePitchInBytes = srcRowPitchInBytes * p->srcArray->height; - } else { - ShouldNotReachHere(); - } - - void* dstPtr = nullptr; - size_t dstElementSizeInBytes = sizeof(unsigned char); - size_t dstRowPitchInBytes = 0; - size_t dstSlicePitchInBytes = 0; - if (p->dstMemoryType == hipMemoryTypeHost) { - dstPtr = p->dstHost; - dstRowPitchInBytes = p->dstPitch; - dstSlicePitchInBytes = dstRowPitchInBytes * p->dstHeight; - } else if ((p->dstMemoryType == hipMemoryTypeDevice) || - (p->dstMemoryType == hipMemoryTypeUnified)) { - dstPtr = p->dstDevice; - dstRowPitchInBytes = p->dstPitch; - dstSlicePitchInBytes = dstRowPitchInBytes * p->dstHeight; - } else if (p->dstMemoryType == hipMemoryTypeArray) { - dstPtr = p->dstArray->data; - getByteSizeFromChannelFormatKind(p->dstArray->desc.f, &dstElementSizeInBytes); - dstElementSizeInBytes *= p->dstArray->NumChannels; - dstRowPitchInBytes = dstElementSizeInBytes * p->dstArray->width; - dstSlicePitchInBytes = dstRowPitchInBytes * p->dstArray->height; - } else { - ShouldNotReachHere(); - } - - // For HIP arrays, srcXInBytes must be evenly divisible by the array element size. - if ((p->srcMemoryType == hipMemoryTypeArray) && - ((p->srcXInBytes % srcElementSizeInBytes) != 0)) { - return hipErrorInvalidValue; - } - - // If specified, srcPitch must be greater than or equal to WidthInBytes + srcXInBytes - if ((p->srcMemoryType != hipMemoryTypeArray) && - (p->srcPitch < (p->WidthInBytes + p->srcXInBytes))) { - return hipErrorInvalidValue; - } - - // If specified, srcHeight must be greater than or equal to Height + srcY - if ((p->srcMemoryType != hipMemoryTypeArray) && - (p->srcHeight < (p->Height + p->srcY))) { - return hipErrorInvalidValue; - } - - // For HIP arrays, dstXInBytes must be evenly divisible by the array element size. - if ((p->dstMemoryType == hipMemoryTypeArray) && - ((p->dstXInBytes % dstElementSizeInBytes) != 0)) { - return hipErrorInvalidValue; - } - - // If specified, srcPitch must be greater than or equal to WidthInBytes + srcXInBytes - if ((p->dstMemoryType != hipMemoryTypeArray) && - (p->dstPitch < (p->WidthInBytes + p->dstXInBytes))) { - return hipErrorInvalidValue; - } - - // If specified, srcHeight must be greater than or equal to Height + srcY - if ((p->dstMemoryType != hipMemoryTypeArray) && - (p->dstHeight < (p->Height + p->dstY))) { - return hipErrorInvalidValue; - } - - // The srcLOD and dstLOD members of the CUDA_MEMCPY3D structure must be set to 0. - if ((p->srcLOD != 0) || (p->dstLOD != 0)) { - return hipErrorInvalidValue; - } - - size_t region[3]; - region[0] = p->WidthInBytes; - region[1] = p->Height; - region[2] = p->Depth; - - size_t srcOrigin[3]; - srcOrigin[0] = p->srcXInBytes; - srcOrigin[1] = p->srcY; - srcOrigin[2] = p->srcZ; - - size_t dstOrigin[3]; - dstOrigin[0] = p->dstXInBytes; - dstOrigin[1] = p->dstY; - dstOrigin[2] = p->dstZ; - - amd::BufferRect srcRect; - if (!srcRect.create(srcOrigin, region, srcRowPitchInBytes, srcSlicePitchInBytes)) { - return hipErrorInvalidValue; - } - - size_t srcMemoryOffset = 0; - amd::Memory* srcMemory = getMemoryObject(srcPtr, srcMemoryOffset); - amd::Coord3D srcStart(srcRect.start_ + srcMemoryOffset, 0, 0); - - amd::BufferRect dstRect; - if (!dstRect.create(dstOrigin, region, dstRowPitchInBytes, dstSlicePitchInBytes)) { - return hipErrorInvalidValue; - } - - size_t dstMemoryOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dstPtr, dstMemoryOffset); - amd::Coord3D dstStart(dstRect.start_ + dstMemoryOffset, 0, 0); - - amd::Command* command = nullptr; - amd::Command::EventWaitList waitList; - amd::HostQueue* queue = hip::getQueue(stream); - amd::Coord3D regionSize(region[0], region[1], region[2]); - - if (((srcMemory == nullptr) && (dstMemory == nullptr)) || - (p->kind == hipMemcpyHostToHost)) { - memcpy(dstPtr, srcPtr, region[0] * region[1] * region[2]); - return hipSuccess; - } else if ((srcMemory == nullptr) && (dstMemory != nullptr)) { - command = new amd::WriteMemoryCommand(*queue, CL_COMMAND_WRITE_BUFFER_RECT, waitList, - *dstMemory->asBuffer(), srcStart, regionSize, srcPtr, srcRect, dstRect); - } else if ((srcMemory != nullptr) && (dstMemory == nullptr)) { - command = new amd::ReadMemoryCommand(*queue, CL_COMMAND_READ_BUFFER_RECT, waitList, - *srcMemory->asBuffer(), srcStart, regionSize, dstPtr, srcRect, dstRect); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_BUFFER_RECT, waitList, - *srcMemory->asBuffer(),*dstMemory->asBuffer(), srcStart, dstStart, regionSize, - srcRect, dstRect); - } - - if (command == nullptr) { - return hipErrorOutOfMemory; - } - - command->enqueue(); - if (!isAsync) { - command->awaitCompletion(); - } - command->release(); - - return hipSuccess; -} - -hipError_t ihipMemcpy3D_V2(const struct hipMemcpy3DParms* p, hipStream_t stream, bool isAsync = false) { - void* srcPtr = nullptr; - size_t srcElementSizeInBytes = sizeof(unsigned char); - size_t srcRowPitchInBytes = 0; - size_t srcSlicePitchInBytes = 0; - if ((p->srcArray != nullptr) && (p->srcPtr.ptr == nullptr)) { - srcPtr = p->srcArray->data; - getByteSizeFromChannelFormatKind(p->srcArray->desc.f, &srcElementSizeInBytes); - srcElementSizeInBytes *= p->srcArray->NumChannels; - srcRowPitchInBytes = srcElementSizeInBytes * p->srcArray->width; - srcSlicePitchInBytes = srcRowPitchInBytes * p->srcArray->height; - } else if ((p->srcArray == nullptr) && (p->srcPtr.ptr != nullptr)) { - srcPtr = p->srcPtr.ptr; - srcRowPitchInBytes = p->srcPtr.pitch; - srcSlicePitchInBytes = srcRowPitchInBytes * p->srcPtr.ysize; - } else { - ShouldNotReachHere(); - } - - void* dstPtr = nullptr; - size_t dstElementSizeInBytes = sizeof(unsigned char); - size_t dstRowPitchInBytes = 0; - size_t dstSlicePitchInBytes = 0; - if ((p->dstArray != nullptr) && (p->dstPtr.ptr == nullptr)) { - dstPtr = p->dstArray->data; - getByteSizeFromChannelFormatKind(p->dstArray->desc.f, &dstElementSizeInBytes); - dstElementSizeInBytes *= p->dstArray->NumChannels; - dstRowPitchInBytes = dstElementSizeInBytes * p->dstArray->width; - dstSlicePitchInBytes = dstRowPitchInBytes * p->dstArray->height; - } else if ((p->dstArray == nullptr) && (p->dstPtr.ptr != nullptr)) { - dstPtr = p->dstPtr.ptr; - dstRowPitchInBytes = p->srcPtr.pitch; - dstSlicePitchInBytes = dstRowPitchInBytes * p->dstPtr.ysize; - } else { - ShouldNotReachHere(); - } - - // If the source and destination are both arrays, they must have the same element size. - if (((p->srcArray != nullptr) && (p->dstArray != nullptr)) && - (srcElementSizeInBytes != dstElementSizeInBytes)) { - return hipErrorInvalidValue; - } - - // If a HIP array is participating in the copy, the extent is defined in terms of that array's elements. - // If no HIP array is participating in the copy, the extent is defined in elements of unsigned char. - size_t region[3]; - if (p->srcArray != nullptr) { - region[0] = srcRowPitchInBytes; - } else if (p->dstArray != nullptr) { - region[0] = dstRowPitchInBytes; - } else { - region[0] = sizeof(unsigned char) * p->extent.width; - } - region[1] = p->extent.height; - region[2] = p->extent.depth; - - // The offset into the object is defined in units of the object's elements. - size_t srcOrigin[3]; - srcOrigin[0] = srcElementSizeInBytes * p->srcPos.x; - srcOrigin[1] = p->srcPos.y; - srcOrigin[2] = p->srcPos.z; - - amd::BufferRect srcRect; - if (!srcRect.create(srcOrigin, region, srcRowPitchInBytes, srcSlicePitchInBytes)) { - return hipErrorInvalidValue; - } - - size_t srcMemoryOffset = 0; - amd::Memory* srcMemory = getMemoryObject(srcPtr, srcMemoryOffset); - amd::Coord3D srcStart(srcRect.start_ + srcMemoryOffset, 0, 0); - - size_t dstOrigin[3]; - dstOrigin[0] = dstElementSizeInBytes * p->dstPos.x; - dstOrigin[1] = p->dstPos.y; - dstOrigin[2] = p->dstPos.z; - - amd::BufferRect dstRect; - if (!dstRect.create(dstOrigin, region, dstRowPitchInBytes, dstSlicePitchInBytes)) { - return hipErrorInvalidValue; - } - - size_t dstMemoryOffset = 0; - amd::Memory* dstMemory = getMemoryObject(dstPtr, dstMemoryOffset); - amd::Coord3D dstStart(dstRect.start_ + dstMemoryOffset, 0, 0); - - amd::Command* command = nullptr; - amd::Command::EventWaitList waitList; - amd::HostQueue* queue = hip::getQueue(stream); - amd::Coord3D regionSize(region[0], region[1], region[2]); - - if (((srcMemory == nullptr) && (dstMemory == nullptr)) || - (p->kind == hipMemcpyHostToHost)) { - memcpy(dstPtr, srcPtr, region[0] * region[1] * region[2]); - return hipSuccess; - } else if ((srcMemory == nullptr) && (dstMemory != nullptr)) { - command = new amd::WriteMemoryCommand(*queue, CL_COMMAND_WRITE_BUFFER_RECT, waitList, - *dstMemory->asBuffer(), srcStart, regionSize, srcPtr, srcRect, dstRect); - } else if ((srcMemory != nullptr) && (dstMemory == nullptr)) { - command = new amd::ReadMemoryCommand(*queue, CL_COMMAND_READ_BUFFER_RECT, waitList, - *srcMemory->asBuffer(), srcStart, regionSize, dstPtr, srcRect, dstRect); - } else if ((srcMemory != nullptr) && (dstMemory != nullptr)) { - command = new amd::CopyMemoryCommand(*queue, CL_COMMAND_COPY_BUFFER_RECT, waitList, - *srcMemory->asBuffer(),*dstMemory->asBuffer(), srcStart, dstStart, regionSize, - srcRect, dstRect); - } - - if (command == nullptr) { - return hipErrorOutOfMemory; - } - - command->enqueue(); - if (!isAsync) { - command->awaitCompletion(); - } - command->release(); - - return hipSuccess; -} - -hipError_t ihipMemcpy3D(const struct hipMemcpy3DParms* p, hipStream_t stream, bool isAsync = false) { - // Having src and dst be an array is ambigous, since we can't tell if the user intended to call hipMemcpy3D_V1() or hipMemcpy3D_V2(). +hipError_t ihipMemcpy3D(const hipMemcpy3DParms* p, + hipStream_t stream, + bool isAsync = false) { + // Having src and dst be an array is ambigous, since we can't tell + // if the user intended to call runtime or driver version of hipMemcpy3D(). // For now hope that we never encounter this case. assert((p->srcArray == nullptr) || (p->dstArray == nullptr)); - // When calling hipMemcpy3D_V1(), the user must specify - // one of srcHost, srcDevice or srcArray and - // one of dstHost, dstDevice or dstArray. - if (((p->srcHost != nullptr) || (p->srcDevice != nullptr) || (p->srcArray != nullptr)) && - ((p->dstHost != nullptr) || (p->dstDevice != nullptr) || (p->dstArray != nullptr))) { - return ihipMemcpy3D_V1(p, stream, isAsync); + // Now we need to patch the user provided struct if they intended on calling the runtime version. + hipMemcpy3DParms pCopy = {}; + std::memcpy(&pCopy, p, sizeof(hipMemcpy3DParms)); + + if ((p->srcPtr.ptr != nullptr) || (p->dstPtr.ptr != nullptr)) { + pCopy.WidthInBytes = p->extent.width; + pCopy.Height = p->extent.height; + pCopy.Depth = p->extent.depth; } - // When calling hipMemcpy3D_V2(), the user must specify - // one of srcArray or srcPtr and - // one of dstArray or dstPtr. - if (((p->srcArray != nullptr) || (p->srcPtr.ptr != nullptr)) && - ((p->dstArray != nullptr) || (p->dstPtr.ptr != nullptr))) { - return ihipMemcpy3D_V2(p, stream, isAsync); + if (p->srcPtr.ptr != nullptr) { + pCopy.srcXInBytes = p->srcPos.x; + pCopy.srcY = p->srcPos.y; + pCopy.srcZ = p->srcPos.z; + pCopy.srcLOD = 0; + pCopy.srcMemoryType = std::get<0>(hip::getMemoryType(p->kind)); + pCopy.srcHost = p->srcPtr.ptr; + pCopy.srcDevice = p->srcPtr.ptr; + pCopy.srcPitch = p->srcPtr.pitch; + pCopy.srcHeight = p->srcPtr.ysize; + + if (p->dstArray != nullptr) { + pCopy.dstMemoryType = hipMemoryTypeArray; + // When reffering to array memory, hipExtent::width is in elements. + pCopy.WidthInBytes *= hip::getElementSize(p->dstArray->Format); + } } - // If we got here, then the user specified an invalid combination of src/dst parameters. - return hipErrorInvalidValue; + if (p->dstPtr.ptr != nullptr) { + pCopy.dstXInBytes = p->dstPos.x; + pCopy.dstY = p->dstPos.y; + pCopy.dstZ = p->dstPos.z; + pCopy.dstLOD = 0; + pCopy.dstMemoryType = std::get<1>(hip::getMemoryType(p->kind)); + pCopy.dstHost = p->dstPtr.ptr; + pCopy.dstDevice = p->dstPtr.ptr; + pCopy.dstPitch = p->dstPtr.pitch; + pCopy.dstHeight = p->dstPtr.ysize; + + if (p->srcArray != nullptr) { + pCopy.srcMemoryType = hipMemoryTypeArray; + // When reffering to array memory, hipExtent::width is in elements. + pCopy.WidthInBytes *= hip::getElementSize(p->srcArray->Format); + } + } + + return ihipMemcpyParam3D(&pCopy, stream, isAsync); } -hipError_t hipMemcpy3D(const struct hipMemcpy3DParms* p) { +hipError_t hipMemcpy3D(const hipMemcpy3DParms* p) { HIP_INIT_API(hipMemcpy3D, p); HIP_RETURN(ihipMemcpy3D(p, nullptr)); } -hipError_t hipMemcpy3DAsync(const struct hipMemcpy3DParms* p, hipStream_t stream) { +hipError_t hipMemcpy3DAsync(const hipMemcpy3DParms* p, hipStream_t stream) { HIP_INIT_API(hipMemcpy3DAsync, p, stream); HIP_RETURN(ihipMemcpy3D(p, stream, true)); @@ -1655,16 +1816,6 @@ hipError_t hipIpcCloseMemHandle(void* dev_ptr) { HIP_RETURN(hipSuccess); } -hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f) { - hipChannelFormatDesc cd; - cd.x = x; - cd.y = y; - cd.z = z; - cd.w = w; - cd.f = f; - return cd; -} - hipError_t hipHostGetDevicePointer(void** devicePointer, void* hostPointer, unsigned flags) { HIP_INIT_API(hipHostGetDevicePointer, devicePointer, hostPointer, flags); @@ -1674,7 +1825,7 @@ hipError_t hipHostGetDevicePointer(void** devicePointer, void* hostPointer, unsi if (!memObj) { HIP_RETURN(hipErrorInvalidValue); } - *devicePointer = reinterpret_cast(memObj->getDeviceMemory(*hip::getCurrentDevice()->devices()[0])->virtualAddress() + offset); +*devicePointer = reinterpret_cast(memObj->getDeviceMemory(*hip::getCurrentDevice()->devices()[0])->virtualAddress() + offset); HIP_RETURN(hipSuccess); } @@ -1710,3 +1861,167 @@ hipError_t hipPointerGetAttributes(hipPointerAttribute_t* attributes, const void HIP_RETURN(hipErrorInvalidValue); } + +hipError_t hipArrayDestroy(hipArray* array) { + HIP_INIT_API(hipArrayDestroy, array); + + HIP_RETURN(ihipArrayDestroy(array)); +} + +hipError_t hipArray3DGetDescriptor(HIP_ARRAY3D_DESCRIPTOR* pArrayDescriptor, + hipArray* array) { + HIP_INIT_API(hipArray3DGetDescriptor, pArrayDescriptor, array); + + assert(false && "Unimplemented"); + + HIP_RETURN(hipSuccess); +} + +hipError_t hipArrayGetDescriptor(HIP_ARRAY_DESCRIPTOR* pArrayDescriptor, + hipArray* array) { + HIP_INIT_API(hipArrayGetDescriptor, pArrayDescriptor, array); + + assert(false && "Unimplemented"); + + HIP_RETURN(hipSuccess); +} + +hipError_t hipMemcpyParam2DAsync(const hip_Memcpy2D* pCopy, + hipStream_t stream) { + HIP_INIT_API(hipMemcpyParam2D, pCopy); + + HIP_RETURN(ihipMemcpyParam2D(pCopy, stream, true)); +} + +hipError_t ihipMemcpy2DArrayToArray(hipArray_t dst, size_t wOffsetDst, size_t hOffsetDst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { + hip_Memcpy2D desc = {}; + + desc.srcXInBytes = wOffsetSrc; + desc.srcY = hOffsetSrc; + desc.srcMemoryType = hipMemoryTypeArray; + desc.srcHost = nullptr; + desc.srcDevice = nullptr; + desc.srcArray = const_cast(src); + desc.srcPitch = 0; // Ignored. + + desc.dstXInBytes = wOffsetDst; + desc.dstY = hOffsetDst; + desc.dstMemoryType = hipMemoryTypeArray; + desc.dstHost = nullptr; + desc.dstDevice = nullptr; + desc.dstArray = dst; + desc.dstPitch = 0; // Ignored. + + desc.WidthInBytes = width; + desc.Height = height; + + return ihipMemcpyParam2D(&desc, stream, isAsync); +} + +hipError_t hipMemcpy2DArrayToArray(hipArray_t dst, size_t wOffsetDst, size_t hOffsetDst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind) { + HIP_INIT_API(hipMemcpy2DArrayToArray, dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind); + + HIP_RETURN(ihipMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind, nullptr)); +} + +hipError_t hipMemcpyArrayToArray(hipArray_t dst, size_t wOffsetDst, size_t hOffsetDst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind) { + HIP_INIT_API(hipMemcpyArrayToArray, dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind); + + HIP_RETURN(ihipMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind, nullptr)); +} + +hipError_t hipMemcpy2DFromArray(void* dst, size_t dpitch, hipArray_const_t src, size_t wOffsetSrc, size_t hOffset, size_t width, size_t height, hipMemcpyKind kind) { + HIP_INIT_API(hipMemcpy2DFromArray, dst, dpitch, src, wOffsetSrc, hOffset, width, height, kind); + + HIP_RETURN(ihipMemcpy2DFromArray(dst, dpitch, src, wOffsetSrc, hOffset, width, height, kind, nullptr)); +} + +hipError_t hipMemcpy2DFromArrayAsync(void* dst, size_t dpitch, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream) { + HIP_INIT_API(hipMemcpy2DFromArrayAsync, dst, dpitch, src, wOffsetSrc, hOffsetSrc, width, height, kind, stream); + + HIP_RETURN(ihipMemcpy2DFromArray(dst, dpitch, src, wOffsetSrc, hOffsetSrc, width, height, kind, stream, true)); +} + +hipError_t hipMemcpyFromArrayAsync(void* dst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, hipMemcpyKind kind, hipStream_t stream) { + HIP_INIT_API(hipMemcpyFromArrayAsync, dst, src, wOffsetSrc, hOffsetSrc, count, kind, stream); + + if (src == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + const size_t arrayHeight = (src->height != 0) ? src->height : 1; + const size_t widthInBytes = count / arrayHeight; + + const size_t height = (count / src->width) / (hip::getElementSize(src->Format) * src->NumChannels); + + HIP_RETURN(ihipMemcpy2DFromArray(dst, 0 /* dpitch */, src, wOffsetSrc, hOffsetSrc, widthInBytes, height, kind, stream, true)); +} + +hipError_t hipMemcpy2DToArrayAsync(hipArray* dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream) { + HIP_INIT_API(hipMemcpy2DToArrayAsync, dst, wOffset, hOffset, src, spitch, width, height, kind); + + HIP_RETURN(ihipMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind, stream, true)); +} + +hipError_t hipMemcpyToArrayAsync(hipArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, hipMemcpyKind kind, hipStream_t stream) { + HIP_INIT_API(hipMemcpyToArrayAsync, dst, wOffset, hOffset, src, count, kind); + + if (dst == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + const size_t arrayHeight = (dst->height != 0) ? dst->height : 1; + const size_t widthInBytes = count / arrayHeight; + + const size_t height = (count / dst->width) / (hip::getElementSize(dst->Format) * dst->NumChannels); + + HIP_RETURN(ihipMemcpy2DToArray(dst, wOffset, hOffset, src, 0 /* spitch */, widthInBytes, height, kind, stream, true)); +} + +hipError_t hipMemcpyAtoA(hipArray* dstArray, + size_t dstOffset, + hipArray* srcArray, + size_t srcOffset, + size_t ByteCount) { + HIP_INIT_API(hipMemcpyAtoA, dstArray, dstOffset, srcArray, srcOffset, ByteCount); + + HIP_RETURN(ihipMemcpyAtoA(srcArray, dstArray, {srcOffset, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, nullptr)); +} + +hipError_t hipMemcpyAtoD(hipDeviceptr_t dstDevice, + hipArray* srcArray, + size_t srcOffset, + size_t ByteCount) { + HIP_INIT_API(hipMemcpyAtoD, dstDevice, srcArray, srcOffset, ByteCount); + + HIP_RETURN(ihipMemcpyAtoD(srcArray, dstDevice, {srcOffset, 0, 0}, {0, 0, 0}, {ByteCount, 1, 1}, 0, 0, nullptr)); +} + +hipError_t hipMemcpyAtoHAsync(void* dstHost, + hipArray* srcArray, + size_t srcOffset, + size_t ByteCount, + hipStream_t stream) { + HIP_INIT_API(hipMemcpyAtoHAsync, dstHost, srcArray, srcOffset, ByteCount, stream); + + HIP_RETURN(ihipMemcpyAtoH(srcArray, dstHost, {srcOffset, 0, 0}, {0, 0, 0}, {ByteCount, 1, 1}, 0, 0, stream, true)); +} + +hipError_t hipMemcpyDtoA(hipArray* dstArray, + size_t dstOffset, + hipDeviceptr_t srcDevice, + size_t ByteCount) { + HIP_INIT_API(hipMemcpyDtoA, dstArray, dstOffset, srcDevice, ByteCount); + + HIP_RETURN(ihipMemcpyDtoA(srcDevice, dstArray, {0, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, 0, 0, nullptr)); +} + +hipError_t hipMemcpyHtoAAsync(hipArray* dstArray, + size_t dstOffset, + const void* srcHost, + size_t ByteCount, + hipStream_t stream) { + HIP_INIT_API(hipMemcpyHtoAAsync, dstArray, dstOffset, srcHost, ByteCount, stream); + + HIP_RETURN(ihipMemcpyHtoA(srcHost, dstArray, {0, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, 0, 0, stream, true)); +} \ No newline at end of file diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 98d4a773f4..634c83557c 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -21,6 +21,7 @@ #include #include #include "hip_internal.hpp" +#include "hip_conversions.hpp" #include "platform/sampler.hpp" namespace hip { @@ -30,777 +31,1153 @@ namespace hip { amd::Image* image; amd::Sampler* sampler; hipResourceDesc resDesc; + hipTextureDesc texDesc; + hipResourceViewDesc resViewDesc; + + TextureObject(amd::Image* image_, + amd::Sampler* sampler_, + const hipResourceDesc& resDesc_, + const hipTextureDesc& texDesc_, + const hipResourceViewDesc& resViewDesc_) : + image(image_), + sampler(sampler_), + resDesc(resDesc_), + texDesc(texDesc_), + resViewDesc(resViewDesc_) { + amd::Context& context = *hip::getCurrentDevice()->asContext(); + amd::Device& device = *context.devices()[0]; + + device::Memory* imageMem = image->getDeviceMemory(device); + std::memcpy(imageSRD, imageMem->cpuSrd(), sizeof(imageSRD)); + + device::Sampler* samplerMem = sampler->getDeviceSampler(device); + std::memcpy(samplerSRD, samplerMem->hwState(), sizeof(samplerSRD)); + } }; }; -void getDrvChannelOrderAndType(const enum hipArray_Format Format, unsigned int NumChannels, - cl_channel_order* channelOrder, - cl_channel_type* channelType) { - switch (Format) { - case HIP_AD_FORMAT_UNSIGNED_INT8: - *channelType = CL_UNSIGNED_INT8; - break; - case HIP_AD_FORMAT_UNSIGNED_INT16: - *channelType = CL_UNSIGNED_INT16; - break; - case HIP_AD_FORMAT_UNSIGNED_INT32: - *channelType = CL_UNSIGNED_INT32; - break; - case HIP_AD_FORMAT_SIGNED_INT8: - *channelType = CL_SIGNED_INT8; - break; - case HIP_AD_FORMAT_SIGNED_INT16: - *channelType = CL_SIGNED_INT16; - break; - case HIP_AD_FORMAT_SIGNED_INT32: - *channelType = CL_SIGNED_INT32; - break; - case HIP_AD_FORMAT_HALF: - *channelType = CL_HALF_FLOAT; - break; - case HIP_AD_FORMAT_FLOAT: - *channelType = CL_FLOAT; - break; - default: - break; +amd::Image* ihipImageCreate(const cl_channel_order channelOrder, + const cl_channel_type channelType, + const cl_mem_object_type imageType, + const size_t imageWidth, + const size_t imageHeight, + const size_t imageDepth, + const size_t imageArraySize, + const size_t imageRowPitch, + const size_t imageSlicePitch, + const uint32_t numMipLevels, + amd::Memory* buffer); + +hipError_t ihipCreateTextureObject(hipTextureObject_t* pTexObject, + const hipResourceDesc* pResDesc, + const hipTextureDesc* pTexDesc, + const hipResourceViewDesc* pResViewDesc) { + amd::Device* device = hip::getCurrentDevice()->devices()[0]; + const device::Info& info = device->info(); + + // pResViewDesc can only be specified if the type of resource is a HIP array or a HIP mipmapped array. + if ((pResViewDesc != nullptr) && + ((pResDesc->resType != hipResourceTypeArray) && (pResDesc->resType != hipResourceTypeMipmappedArray))) { + return hipErrorInvalidValue; } - if (NumChannels == 4) { - *channelOrder = CL_RGBA; - } else if (NumChannels == 2) { - *channelOrder = CL_RG; - } else if (NumChannels == 1) { - *channelOrder = CL_R; - } -} - -void setDescFromChannelType(cl_channel_type channelType, hipChannelFormatDesc* desc) { - - memset(desc, 0x00, sizeof(hipChannelFormatDesc)); - - switch (channelType) { - case CL_SIGNED_INT8: - case CL_SIGNED_INT16: - case CL_SIGNED_INT32: - desc->f = hipChannelFormatKindSigned; - break; - case CL_UNSIGNED_INT8: - case CL_UNSIGNED_INT16: - case CL_UNSIGNED_INT32: - desc->f = hipChannelFormatKindUnsigned; - break; - case CL_HALF_FLOAT: - case CL_FLOAT: - desc->f = hipChannelFormatKindFloat; - break; - default: - desc->f = hipChannelFormatKindNone; - break; + // If hipResourceDesc::resType is set to hipResourceTypeArray, + // hipResourceDesc::res::array::array must be set to a valid HIP array handle. + if ((pResDesc->resType == hipResourceTypeArray) && + (pResDesc->res.array.array == nullptr)) { + return hipErrorInvalidValue; } - switch (channelType) { - case CL_SIGNED_INT8: - case CL_UNSIGNED_INT8: - desc->x = 8; - break; - case CL_SIGNED_INT16: - case CL_UNSIGNED_INT16: - case CL_HALF_FLOAT: - desc->x = 16; - break; - case CL_SIGNED_INT32: - case CL_UNSIGNED_INT32: - case CL_FLOAT: - desc->x = 32; - break; - default: - desc->x = 0; - break; + // If hipResourceDesc::resType is set to hipResourceTypeMipmappedArray, + // hipResourceDesc::res::mipmap::mipmap must be set to a valid HIP mipmapped array handle + // and hipTextureDesc::normalizedCoords must be set to true. + if ((pResDesc->resType == hipResourceTypeMipmappedArray) && + ((pResDesc->res.mipmap.mipmap == nullptr) || (pTexDesc->normalizedCoords == 0))) { + return hipErrorInvalidValue; } -} -void getChannelOrderAndType(const hipChannelFormatDesc& desc, enum hipTextureReadMode readMode, - cl_channel_order* channelOrder, cl_channel_type* channelType) { - if (desc.x != 0 && desc.y != 0 && desc.z != 0 && desc.w != 0) { - *channelOrder = CL_RGBA; - } else if (desc.x != 0 && desc.y != 0 && desc.z != 0 && desc.w == 0) { - *channelOrder = CL_RGB; - } else if (desc.x != 0 && desc.y != 0 && desc.z == 0 && desc.w == 0) { - *channelOrder = CL_RG; - } else if (desc.x != 0 && desc.y == 0 && desc.z == 0 && desc.w == 0) { - *channelOrder = CL_R; - } else { - } + // If hipResourceDesc::resType is set to hipResourceTypeLinear, + // hipResourceDesc::res::linear::devPtr must be set to a valid device pointer, that is aligned to hipDeviceProp::textureAlignment. + // The total number of elements in the linear address range cannot exceed hipDeviceProp::maxTexture1DLinear. + if ((pResDesc->resType == hipResourceTypeLinear) && + ((pResDesc->res.linear.devPtr == nullptr) || + (!amd::isMultipleOf(pResDesc->res.linear.devPtr, info.imageBaseAddressAlignment_)) || + (pResDesc->res.linear.sizeInBytes >= info.imageMaxBufferSize_))) { + return hipErrorInvalidValue; + } - switch (desc.f) { - case hipChannelFormatKindUnsigned: - switch (desc.x) { - case 32: - *channelType = CL_UNSIGNED_INT32; - break; - case 16: - *channelType = readMode == hipReadModeNormalizedFloat - ? CL_UNORM_INT16 - : CL_UNSIGNED_INT16; - break; - case 8: - *channelType = readMode == hipReadModeNormalizedFloat - ? CL_UNORM_INT8 - : CL_UNSIGNED_INT8; - break; - default: - *channelType = CL_UNSIGNED_INT32; - } - break; - case hipChannelFormatKindSigned: - switch (desc.x) { - case 32: - *channelType = CL_SIGNED_INT32; - break; - case 16: - *channelType = readMode == hipReadModeNormalizedFloat - ? CL_SNORM_INT16 - : CL_SIGNED_INT16; - break; - case 8: - *channelType = readMode == hipReadModeNormalizedFloat - ? CL_SNORM_INT8 - : CL_SIGNED_INT8; - break; - default: - *channelType = CL_SIGNED_INT32; - } - break; - case hipChannelFormatKindFloat: - switch (desc.x) { - case 32: - *channelType = CL_FLOAT; - break; - case 16: - *channelType = CL_HALF_FLOAT; - break; - case 8: - break; - default: - *channelType = CL_FLOAT; - } - break; - case hipChannelFormatKindNone: - default: - break; - } -} + // If hipResourceDesc::resType is set to hipResourceTypePitch2D, + // hipResourceDesc::res::pitch2D::devPtr must be set to a valid device pointer, that is aligned to hipDeviceProp::textureAlignment. + // hipResourceDesc::res::pitch2D::width and hipResourceDesc::res::pitch2D::height specify the width and height of the array in elements, + // and cannot exceed hipDeviceProp::maxTexture2DLinear[0] and hipDeviceProp::maxTexture2DLinear[1] respectively. + // hipResourceDesc::res::pitch2D::pitchInBytes specifies the pitch between two rows in bytes and has to be aligned to hipDeviceProp::texturePitchAlignment. + // Pitch cannot exceed hipDeviceProp::maxTexture2DLinear[2]. + if ((pResDesc->resType == hipResourceTypePitch2D) && + ((pResDesc->res.pitch2D.devPtr == nullptr) || + (!amd::isMultipleOf(pResDesc->res.pitch2D.devPtr, info.imageBaseAddressAlignment_)) || + (pResDesc->res.pitch2D.width >= info.image2DMaxWidth_) || + (pResDesc->res.pitch2D.height >= info.image2DMaxHeight_) || + (!amd::isMultipleOf(pResDesc->res.pitch2D.pitchInBytes, info.imagePitchAlignment_)))) { + // TODO check pitch limits. + return hipErrorInvalidValue; + } -void getByteSizeFromChannelFormatKind(enum hipChannelFormatKind channelFormatKind, size_t* byteSize) { - switch (channelFormatKind) - { - case hipChannelFormatKindSigned: - *byteSize = sizeof(int); - break; - case hipChannelFormatKindUnsigned: - *byteSize = sizeof(unsigned int); - break; - case hipChannelFormatKindFloat: - *byteSize = sizeof(float); - break; - case hipChannelFormatKindNone: - *byteSize = sizeof(size_t); - break; - default: - *byteSize = 1; - break; - } -} + // Mipmaps are currently not supported. + if (pResDesc->resType == hipResourceTypeMipmappedArray) { + return hipErrorNotSupported; + } + // We don't program the border_color_ptr field in the HW sampler SRD. + if (pTexDesc->addressMode[0] == hipAddressModeBorder) { + return hipErrorNotSupported; + } + // We don't program the force_degamma/skip_degamma fields in the HW sampler SRD. + if (pTexDesc->sRGB == 1) { + return hipErrorNotSupported; + } + // We don't program the max_ansio_ratio field in the the HW sampler SRD. + if (pTexDesc->maxAnisotropy != 0) { + return hipErrorNotSupported; + } + // We don't program the lod_bias field in the HW sampler SRD. + if (pTexDesc->mipmapLevelBias != 0) { + return hipErrorNotSupported; + } + // We don't program the min_lod field in the HW sampler SRD. + if (pTexDesc->minMipmapLevelClamp != 0) { + return hipErrorNotSupported; + } + // We don't program the max_lod field in the HW sampler SRD. + if (pTexDesc->maxMipmapLevelClamp != 0) { + return hipErrorNotSupported; + } + + // TODO VDI assumes all dimensions have the same addressing mode. + cl_addressing_mode addressMode = CL_ADDRESS_NONE; + // If hipTextureDesc::normalizedCoords is set to zero, + // hipAddressModeWrap and hipAddressModeMirror won't be supported + // and will be switched to hipAddressModeClamp. + if ((pTexDesc->normalizedCoords == 0) && + ((pTexDesc->addressMode[0] == hipAddressModeWrap) || (pTexDesc->addressMode[0] == hipAddressModeMirror))) { + addressMode = hip::getCLAddressingMode(hipAddressModeClamp); + } + // hipTextureDesc::addressMode is ignored if hipResourceDesc::resType is hipResourceTypeLinear + else if (pResDesc->resType != hipResourceTypeLinear) { + addressMode = hip::getCLAddressingMode(pTexDesc->addressMode[0]); + } -amd::Sampler* fillSamplerDescriptor(enum hipTextureAddressMode addressMode, - enum hipTextureFilterMode filterMode, int normalizedCoords) { #ifndef CL_FILTER_NONE #define CL_FILTER_NONE 0x1142 #endif - uint32_t filter_mode = CL_FILTER_NONE; - switch (filterMode) { - case hipFilterModePoint: - filter_mode = CL_FILTER_NEAREST; - break; - case hipFilterModeLinear: - filter_mode = CL_FILTER_LINEAR; - break; + cl_filter_mode filterMode = CL_FILTER_NONE; +#undef CL_FILTER_NONE + // hipTextureDesc::filterMode is ignored if hipResourceDesc::resType is hipResourceTypeLinear. + if (pResDesc->resType != hipResourceTypeLinear) { + filterMode = hip::getCLFilterMode(pTexDesc->filterMode); } - uint32_t address_mode = CL_ADDRESS_NONE; - switch (addressMode) { - case hipAddressModeWrap: - address_mode = CL_ADDRESS_REPEAT; - break; - case hipAddressModeClamp: - address_mode = CL_ADDRESS_CLAMP; - break; - case hipAddressModeMirror: - address_mode = CL_ADDRESS_MIRRORED_REPEAT; - break; - case hipAddressModeBorder: - address_mode = CL_ADDRESS_CLAMP_TO_EDGE; - break; +#ifndef CL_FILTER_NONE +#define CL_FILTER_NONE 0x1142 +#endif + cl_filter_mode mipFilterMode = CL_FILTER_NONE; +#undef CL_FILTER_NONE + if (pResDesc->resType == hipResourceTypeMipmappedArray) { + mipFilterMode = hip::getCLFilterMode(pTexDesc->mipmapFilterMode); } - amd::Sampler* sampler = new amd::Sampler(*hip::getCurrentDevice()->asContext(), - normalizedCoords == CL_TRUE, - address_mode, filter_mode, CL_FILTER_NONE, 0.f, CL_MAXFLOAT); + + amd::Sampler* sampler = new amd::Sampler(*hip::getCurrentDevice()->asContext(), + pTexDesc->normalizedCoords, + addressMode, + filterMode, + mipFilterMode, + pTexDesc->minMipmapLevelClamp, + pTexDesc->maxMipmapLevelClamp); + if (sampler == nullptr) { - return nullptr; + return hipErrorOutOfMemory; } + if (!sampler->create()) { delete sampler; - return nullptr; - } - return sampler; -} - -hip::TextureObject* ihipCreateTextureObject(const hipResourceDesc& resDesc, amd::Image& image, amd::Sampler& sampler) { - hip::TextureObject* texture; - ihipMalloc(reinterpret_cast(&texture), sizeof(hip::TextureObject), CL_MEM_SVM_FINE_GRAIN_BUFFER); - - if (texture == nullptr) { - return nullptr; - } - - device::Memory* imageMem = image.getDeviceMemory(*hip::getCurrentDevice()->devices()[0]); - memcpy(texture->imageSRD, imageMem->cpuSrd(), sizeof(uint32_t)*HIP_IMAGE_OBJECT_SIZE_DWORD); - texture->image = ℑ - - device::Sampler* devSampler = sampler.getDeviceSampler(*hip::getCurrentDevice()->devices()[0]); - memcpy(texture->samplerSRD, devSampler->hwState(), sizeof(uint32_t)*HIP_SAMPLER_OBJECT_SIZE_DWORD); - texture->sampler = &sampler; - - memcpy(&texture->resDesc, &resDesc, sizeof(hipResourceDesc)); - - return texture; -} - -hipError_t hipCreateTextureObject(hipTextureObject_t* pTexObject, const hipResourceDesc* pResDesc, - const hipTextureDesc* pTexDesc, - const hipResourceViewDesc* pResViewDesc) { - HIP_INIT_API(NONE, pTexObject, pResDesc, pTexDesc, pResViewDesc); - - amd::Device* device = hip::getCurrentDevice()->devices()[0]; - - if (!device->info().imageSupport_) { - HIP_RETURN(hipErrorInvalidValue); + return hipErrorOutOfMemory; } amd::Image* image = nullptr; - - cl_image_format image_format; - getChannelOrderAndType(pResDesc->res.pitch2D.desc, pTexDesc->readMode, - &image_format.image_channel_order, &image_format.image_channel_data_type); - - const amd::Image::Format imageFormat(image_format); - - amd::Memory* memory = nullptr; - size_t offset = 0; - cl_mem_object_type clType; - switch (pResDesc->resType) { - case hipResourceTypeArray: - { - memory = getMemoryObject(pResDesc->res.array.array->data, offset); + case hipResourceTypeArray: { + cl_mem memObj = reinterpret_cast(pResDesc->res.array.array->data); + if (!is_valid(memObj)) { + return hipErrorInvalidValue; + } + image = as_amd(memObj)->asImage(); - getChannelOrderAndType(pResDesc->res.array.array->desc, pTexDesc->readMode, - &image_format.image_channel_order, &image_format.image_channel_data_type); - const amd::Image::Format imageFormat(image_format); - switch (pResDesc->res.array.array->type) { - case hipArrayLayered: - case hipArrayCubemap: - assert(0); - break; - case hipArraySurfaceLoadStore: - case hipArrayTextureGather: - case hipArrayDefault: - default: - switch(pResDesc->res.array.array->textureType) { - case hipTextureType3D: - clType = CL_MEM_OBJECT_IMAGE3D; - image = new (*hip::getCurrentDevice()->asContext()) amd::Image(*memory->asBuffer(), - clType, memory->getMemFlags(), imageFormat, - pResDesc->res.array.array->width, pResDesc->res.array.array->height, - pResDesc->res.array.array->depth, 0, 0); - break; - case hipTextureType2D: - clType = CL_MEM_OBJECT_IMAGE2D; - image = new (*hip::getCurrentDevice()->asContext()) amd::Image(*memory->asBuffer(), - clType, memory->getMemFlags(), imageFormat, - pResDesc->res.array.array->width, pResDesc->res.array.array->height, 1, 0, 0); - break; - default: - break; - } - break; - } + hipTextureReadMode readMode = pTexDesc->readMode; + // 32-bit integer format will not be promoted, regardless of whether or not + // this hipTextureDesc::readMode is set hipReadModeNormalizedFloat is specified. + if ((hip::getElementSize(pResDesc->res.array.array->Format) == 4) && + (pResDesc->res.array.array->Format != HIP_AD_FORMAT_FLOAT)) { + readMode = hipReadModeElementType; + } + + // We need to create an image view if the user requested to use normalized pixel values, + // due to already having the image created with a different format. + if ((pResViewDesc != nullptr) || + (readMode == hipReadModeNormalizedFloat)) { + // TODO VDI currently right now can only change the format of the image. + const cl_channel_order channelOrder = (pResViewDesc != nullptr) ? hip::getCLChannelOrder(hip::getNumChannels(pResViewDesc->format)) : + hip::getCLChannelOrder(pResDesc->res.array.array->NumChannels); + const cl_channel_type channelType = (pResViewDesc != nullptr) ? hip::getCLChannelType(hip::getArrayFormat(pResViewDesc->format), readMode) : + hip::getCLChannelType(pResDesc->res.array.array->Format, readMode); + const amd::Image::Format imageFormat(cl_image_format{channelOrder, channelType}); + if (!imageFormat.isValid()) { + return hipErrorInvalidValue; } - break; - case hipResourceTypeMipmappedArray: - assert(0); - break; - case hipResourceTypeLinear: - { - assert(pResViewDesc == nullptr); - memory = getMemoryObject(pResDesc->res.linear.devPtr, offset); - getChannelOrderAndType(pResDesc->res.linear.desc, pTexDesc->readMode, - &image_format.image_channel_order, &image_format.image_channel_data_type); - const amd::Image::Format imageFormat(image_format); - - image = new (*hip::getCurrentDevice()->asContext()) amd::Image(*memory->asBuffer(), - CL_MEM_OBJECT_IMAGE2D, memory->getMemFlags(), imageFormat, - pResDesc->res.linear.sizeInBytes / imageFormat.getElementSize(), 1, 1, - pResDesc->res.linear.sizeInBytes, 0); + image = image->createView(*hip::getCurrentDevice()->asContext(), imageFormat, nullptr); + if (image == nullptr) { + return hipErrorInvalidValue; } - break; - case hipResourceTypePitch2D: - assert(pResViewDesc == nullptr); - memory = getMemoryObject(pResDesc->res.pitch2D.devPtr, offset); - - image = new (*hip::getCurrentDevice()->asContext()) amd::Image(*memory->asBuffer(), - CL_MEM_OBJECT_IMAGE2D, memory->getMemFlags(), imageFormat, - pResDesc->res.pitch2D.width, pResDesc->res.pitch2D.height, 1, - pResDesc->res.pitch2D.pitchInBytes, 0); - break; - default: HIP_RETURN(hipErrorInvalidValue); - } - - if (!image->create()) { - delete image; - HIP_RETURN(hipErrorOutOfMemory); - } - - amd::Sampler* sampler = fillSamplerDescriptor(pTexDesc->addressMode[0], pTexDesc->filterMode, pTexDesc->normalizedCoords); - - *pTexObject = reinterpret_cast(ihipCreateTextureObject(*pResDesc, *image, *sampler)); - - HIP_RETURN(hipSuccess); -} - -void ihipDestroyTextureObject(hip::TextureObject* texture) { - texture->image->release(); - texture->sampler->release(); - - hipFree(texture); -} - -hipError_t hipDestroyTextureObject(hipTextureObject_t textureObject) { - HIP_INIT_API(NONE, textureObject); - - hip::TextureObject* texture = reinterpret_cast(textureObject); - - ihipDestroyTextureObject(texture); - - HIP_RETURN(hipSuccess); -} - -hipError_t hipGetTextureObjectResourceDesc(hipResourceDesc* pResDesc, - hipTextureObject_t textureObject) { - HIP_INIT_API(NONE, pResDesc, textureObject); - - hip::TextureObject* texture = reinterpret_cast(textureObject); - - if (pResDesc != nullptr && texture != nullptr) { - memcpy(pResDesc, &(texture->resDesc), sizeof(hipResourceDesc)); - } - - HIP_RETURN(hipErrorInvalidValue); -} - -hipError_t hipGetTextureObjectResourceViewDesc(hipResourceViewDesc* pResViewDesc, - hipTextureObject_t textureObject) { - HIP_INIT_API(NONE, pResViewDesc, textureObject); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t hipGetTextureObjectTextureDesc(hipTextureDesc* pTexDesc, - hipTextureObject_t textureObject) { - HIP_INIT_API(NONE, pTexDesc, textureObject); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t ihipBindTexture(cl_mem_object_type type, - size_t* offset, textureReference* tex, const void* devPtr, - const hipChannelFormatDesc& desc, size_t width, size_t height, - size_t pitch) { - if (tex == nullptr) { - return hipErrorInvalidImage; - } - if (hip::getCurrentDevice()) { - cl_image_format image_format; - size_t byteSize; - size_t rowPitch = 0; - size_t depth = 0; - size_t slicePitch = 0; - - getChannelOrderAndType(desc, hipReadModeElementType, - &image_format.image_channel_order, &image_format.image_channel_data_type); - getByteSizeFromChannelFormatKind(desc.f, &byteSize); - const amd::Image::Format imageFormat(image_format); - amd::Memory* memory = getMemoryObject(devPtr, *offset); - - switch (type) { - case CL_MEM_OBJECT_IMAGE3D: - rowPitch = width * byteSize; - depth = pitch; - slicePitch = rowPitch * height; - break; - case CL_MEM_OBJECT_IMAGE2D: - default: - rowPitch = pitch; - depth = 1; - slicePitch = 0; - break; } - - amd::Image* image = new (*hip::getCurrentDevice()->asContext()) amd::Image(*memory->asBuffer(), - type, memory->getMemFlags(), imageFormat, width, height, depth, rowPitch, slicePitch); - if (!image->create()) { - delete image; - return hipErrorOutOfMemory; - } - - *offset = 0; - if (tex->textureObject) { - ihipDestroyTextureObject(reinterpret_cast(tex->textureObject)); - } - amd::Sampler* sampler = fillSamplerDescriptor(tex->addressMode[0], tex->filterMode, tex->normalized); - - hipResourceDesc resDesc; - memset(&resDesc, 0, sizeof(hipResourceDesc)); - switch (type) { - case CL_MEM_OBJECT_IMAGE1D: - resDesc.resType = hipResourceTypeLinear; - resDesc.res.linear.devPtr = const_cast(devPtr); - resDesc.res.linear.desc = desc; - resDesc.res.linear.sizeInBytes = image->getSize(); - break; - case CL_MEM_OBJECT_IMAGE2D: - resDesc.resType = hipResourceTypePitch2D; - resDesc.res.pitch2D.devPtr = const_cast(devPtr); - resDesc.res.pitch2D.desc = desc; - resDesc.res.pitch2D.width = width; - resDesc.res.pitch2D.height = height; - resDesc.res.pitch2D.pitchInBytes = pitch; - break; - case CL_MEM_OBJECT_IMAGE3D: - resDesc.resType = hipResourceTypeArray; - resDesc.res.array.array = (hipArray*)malloc(sizeof(hipArray)); - resDesc.res.array.array->desc = desc; - resDesc.res.array.array->width = width; - resDesc.res.array.array->height = height; - resDesc.res.array.array->depth = depth; - resDesc.res.array.array->Format = tex->format; - resDesc.res.array.array->NumChannels = tex->numChannels; - resDesc.res.array.array->isDrv = false; - resDesc.res.array.array->textureType = hipTextureType3D; - resDesc.res.array.array->data = const_cast(devPtr); - break; - default: - resDesc.resType = hipResourceTypeArray; - resDesc.res.array.array = nullptr; - break; - } - - tex->textureObject = reinterpret_cast(ihipCreateTextureObject(resDesc, *image, *sampler)); - if(type == CL_MEM_OBJECT_IMAGE3D) { - free(resDesc.res.array.array); - } - memset(&resDesc, 0, sizeof(hipResourceDesc)); - return hipSuccess; + break; } - return hipErrorInvalidValue; -} - -hipError_t hipBindTexture(size_t* offset, textureReference* tex, const void* devPtr, - const hipChannelFormatDesc* desc, size_t size) { - HIP_INIT_API(NONE, offset, tex, devPtr, desc, size); - - if (desc == nullptr) { - HIP_RETURN(hipErrorInvalidValue); + case hipResourceTypeMipmappedArray: { + ShouldNotReachHere(); + break; + } + case hipResourceTypeLinear: { + const cl_channel_order channelOrder = hip::getCLChannelOrder(hip::getNumChannels(pResDesc->res.linear.desc)); + const cl_channel_type channelType = hip::getCLChannelType(hip::getArrayFormat(pResDesc->res.linear.desc), pTexDesc->readMode); + const amd::Image::Format imageFormat({channelOrder, channelType}); + const cl_mem_object_type imageType = hip::getCLMemObjectType(pResDesc->resType); + size_t offset = 0; + image = ihipImageCreate(channelOrder, + channelType, + imageType, + (pResDesc->res.linear.sizeInBytes / imageFormat.getElementSize()), /* imageWidth */ + 0, /* imageHeight */ + 0, /* imageDepth */ + 0, /* imageArraySize */ + 0, /* imageRowPitch */ + 0, /* imageSlicePitch */ + 0, /* numMipLevels */ + getMemoryObject(pResDesc->res.linear.devPtr, offset)); + // TODO take care of non-zero offset. + assert(offset == 0); + if (image == nullptr) { + return hipErrorInvalidValue; + } + break; + } + case hipResourceTypePitch2D: { + const cl_channel_order channelOrder = hip::getCLChannelOrder(hip::getNumChannels(pResDesc->res.pitch2D.desc)); + const cl_channel_type channelType = hip::getCLChannelType(hip::getArrayFormat(pResDesc->res.pitch2D.desc), pTexDesc->readMode); + const cl_mem_object_type imageType = hip::getCLMemObjectType(pResDesc->resType); + size_t offset = 0; + image = ihipImageCreate(channelOrder, + channelType, + imageType, + pResDesc->res.pitch2D.width, /* imageWidth */ + pResDesc->res.pitch2D.height, /* imageHeight */ + 0, /* imageDepth */ + 0, /* imageArraySize */ + pResDesc->res.pitch2D.pitchInBytes, /* imageRowPitch */ + 0, /* imageSlicePitch */ + 0, /* numMipLevels */ + getMemoryObject(pResDesc->res.pitch2D.devPtr, offset)); + // TODO take care of non-zero offset. + assert(offset == 0); + if (image == nullptr) { + return hipErrorInvalidValue; + } + break; } - cl_image_format image_format; - getChannelOrderAndType(*desc, hipReadModeElementType, - &image_format.image_channel_order, &image_format.image_channel_data_type); - const amd::Image::Format imageFormat(image_format); - - HIP_RETURN(ihipBindTexture(CL_MEM_OBJECT_IMAGE1D, offset, tex, devPtr, *desc, size / imageFormat.getElementSize(), 1, size)); -} - -hipError_t hipBindTexture2D(size_t* offset, textureReference* tex, const void* devPtr, - const hipChannelFormatDesc* desc, size_t width, size_t height, - size_t pitch) { - HIP_INIT_API(NONE, offset, tex, devPtr, desc, width, height, pitch); - - HIP_RETURN(ihipBindTexture(CL_MEM_OBJECT_IMAGE2D, offset, tex, devPtr, *desc, width, height, pitch)); -} - -hipError_t hipBindTextureToArray(textureReference* tex, hipArray_const_t array, - const hipChannelFormatDesc* desc) { - HIP_INIT_API(NONE, tex, array, desc); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t ihipBindTextureImpl(TlsData* tls, int dim, enum hipTextureReadMode readMode, size_t* offset, - const void* devPtr, const struct hipChannelFormatDesc* desc, - size_t size, textureReference* tex) { - HIP_INIT_API(NONE, dim, readMode, offset, devPtr, size, tex); - - assert(1 == dim); - - HIP_RETURN(ihipBindTexture(CL_MEM_OBJECT_IMAGE1D, offset, tex, devPtr, *desc, size, 1, 0)); -} - -hipError_t ihipBindTextureToArrayImpl(TlsData* tls, int dim, enum hipTextureReadMode readMode, - hipArray_const_t array, - const struct hipChannelFormatDesc& desc, - textureReference* tex) { - HIP_INIT_API(NONE, dim, readMode, &desc, array, tex); - - cl_mem_object_type clType; - size_t offset = 0; - - switch (dim) { - case 1: - clType = CL_MEM_OBJECT_IMAGE1D; - break; - case 2: - clType = CL_MEM_OBJECT_IMAGE2D; - break; - case 3: - case hipTextureType2DLayered: - clType = CL_MEM_OBJECT_IMAGE3D; - break; - default: - HIP_RETURN(hipErrorInvalidValue); } - HIP_RETURN(ihipBindTexture(clType, &offset, tex, array->data, desc, array->width, - array->height, array->depth)); -} - -hipError_t hipBindTextureToMipmappedArray(textureReference* tex, - hipMipmappedArray_const_t mipmappedArray, - const hipChannelFormatDesc* desc) { - HIP_INIT_API(NONE, tex, mipmappedArray, desc); - - assert(0 && "Unimplemented"); - - HIP_RETURN(hipErrorNotSupported); -} - -hipError_t ihipUnbindTextureImpl(const hipTextureObject_t& textureObject) { - - ihipDestroyTextureObject(reinterpret_cast(textureObject)); + void *texObjectBuffer = nullptr; + ihipMalloc(&texObjectBuffer, sizeof(hip::TextureObject), CL_MEM_SVM_FINE_GRAIN_BUFFER); + if (texObjectBuffer == nullptr) { + return hipErrorOutOfMemory; + } + *pTexObject = reinterpret_cast(new (texObjectBuffer) hip::TextureObject{image, sampler, *pResDesc, *pTexDesc, (pResViewDesc != nullptr) ? *pResViewDesc : hipResourceViewDesc{}}); return hipSuccess; } -hipError_t hipUnbindTexture(const textureReference* tex) { - HIP_INIT_API(NONE, tex); +hipError_t hipCreateTextureObject(hipTextureObject_t* pTexObject, + const hipResourceDesc* pResDesc, + const hipTextureDesc* pTexDesc, + const hipResourceViewDesc* pResViewDesc) { + HIP_INIT_API(hipCreateTextureObject, pTexObject, pResDesc, pTexDesc, pResViewDesc); - ihipDestroyTextureObject(reinterpret_cast(tex->textureObject)); - - HIP_RETURN(hipSuccess); + HIP_RETURN(ihipCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc)); } -hipError_t hipGetChannelDesc(hipChannelFormatDesc* desc, hipArray_const_t array) { - HIP_INIT_API(NONE, desc, array); - if (desc != nullptr) { - *desc = array->desc; +hipError_t ihipDestroyTextureObject(hipTextureObject_t texObject) { + if (texObject == nullptr) { + return hipErrorInvalidValue; } - HIP_RETURN(hipSuccess); + hip::TextureObject* hipTexObject = reinterpret_cast(texObject); + const hipResourceType type = hipTexObject->resDesc.resType; + const bool isImageFromBuffer = (type == hipResourceTypeLinear) || (type == hipResourceTypePitch2D); + const bool isImageView = ((type == hipResourceTypeArray) || (type == hipResourceTypeMipmappedArray)) && + !hipTexObject->image->isParent(); + if (isImageFromBuffer || isImageView) { + hipTexObject->image->release(); + } + + // TODO Should call ihipFree() to not polute the api trace. + return hipFree(hipTexObject); } -hipError_t hipGetTextureAlignmentOffset(size_t* offset, const textureReference* tex) { - HIP_INIT_API(NONE, offset, tex); +hipError_t hipDestroyTextureObject(hipTextureObject_t texObject) { + HIP_INIT_API(hipDestroyTextureObject, texObject); - if ((offset == nullptr) || (tex == nullptr)) { + HIP_RETURN(ihipDestroyTextureObject(texObject)); +} + + +hipError_t hipGetTextureObjectResourceDesc(hipResourceDesc* pResDesc, + hipTextureObject_t texObject) { + HIP_INIT_API(hipGetTextureObjectResourceDesc, pResDesc, texObject); + + if ((pResDesc == nullptr) || + (texObject == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } + hip::TextureObject* hipTexObject = reinterpret_cast(texObject); + *pResDesc = hipTexObject->resDesc; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipGetTextureObjectResourceViewDesc(hipResourceViewDesc* pResViewDesc, + hipTextureObject_t texObject) { + HIP_INIT_API(hipGetTextureObjectResourceViewDesc, pResViewDesc, texObject); + + if ((pResViewDesc == nullptr) || + (texObject == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + hip::TextureObject* hipTexObject = reinterpret_cast(texObject); + *pResViewDesc = hipTexObject->resViewDesc; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipGetTextureObjectTextureDesc(hipTextureDesc* pTexDesc, + hipTextureObject_t texObject) { + HIP_INIT_API(hipGetTextureObjectTextureDesc, pTexDesc, texObject); + + if ((pTexDesc == nullptr) || + (texObject == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + hip::TextureObject* hipTexObject = reinterpret_cast(texObject); + *pTexDesc = hipTexObject->texDesc; + + HIP_RETURN(hipSuccess); +} + +hipError_t ihipBindTexture(size_t* offset, + const textureReference* texref, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t size, + hipTextureReadMode readMode) { + if ((offset == nullptr) || + (texref == nullptr) || + (devPtr == nullptr) || + (desc == nullptr)) { + return hipErrorInvalidValue; + } + + // Any previous address or HIP array state associated with the texture reference is superseded by this function. + // Any memory previously bound to hTexRef is unbound. + // No need to check for errors. + ihipDestroyTextureObject(texref->textureObject); + + // If the device memory pointer was returned from hipMalloc(), + // the offset is guaranteed to be 0 and NULL may be passed as the offset parameter. + // TODO enforce alignment on devPtr. + if (offset != nullptr) { + *offset = 0; + } + + hipResourceDesc resDesc = {}; + resDesc.resType = hipResourceTypeLinear; + resDesc.res.linear.devPtr = const_cast(devPtr); + resDesc.res.linear.desc = *desc; + resDesc.res.linear.sizeInBytes = size; + + hipTextureDesc texDesc = hip::getTextureDesc(texref, readMode); + + return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); +} + +hipError_t ihipBindTexture2D(size_t* offset, + const textureReference* texref, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t width, + size_t height, + size_t pitch, + hipTextureReadMode readMode) { + if ((offset == nullptr) || + (texref == nullptr) || + (devPtr == nullptr) || + (desc == nullptr)) { + return hipErrorInvalidValue; + } + + // Any previous address or HIP array state associated with the texture reference is superseded by this function. + // Any memory previously bound to hTexRef is unbound. + // No need to check for errors. + ihipDestroyTextureObject(texref->textureObject); + + // If the device memory pointer was returned from hipMalloc(), + // the offset is guaranteed to be 0 and NULL may be passed as the offset parameter. + // TODO enforce alignment on devPtr. + if (offset != nullptr) { + *offset = 0; + } + + hipResourceDesc resDesc = {}; + resDesc.resType = hipResourceTypePitch2D; + resDesc.res.pitch2D.devPtr = const_cast(devPtr); + resDesc.res.pitch2D.desc = *desc; + resDesc.res.pitch2D.width = width; + resDesc.res.pitch2D.height = height; + resDesc.res.pitch2D.pitchInBytes = pitch; + + hipTextureDesc texDesc = hip::getTextureDesc(texref, readMode); + + return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); +} + +hipError_t hipBindTexture2D(size_t* offset, + textureReference* texref, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t width, + size_t height, + size_t pitch) { + HIP_INIT_API(hipBindTexture2D, offset, texref, devPtr, desc, width, height, pitch); + + // TODO need compiler support to extract the read mode from textureReference. + HIP_RETURN(ihipBindTexture2D(offset, texref, devPtr, desc, width, height, pitch, hipReadModeElementType)); +} + +hipError_t ihipBindTextureToArray(const textureReference* texref, + hipArray_const_t array, + const hipChannelFormatDesc* desc, + hipTextureReadMode readMode) { + if ((texref == nullptr) || + (array == nullptr) || + (desc == nullptr)) { + return hipErrorInvalidValue; + } + + // Any previous address or HIP array state associated with the texture reference is superseded by this function. + // Any memory previously bound to hTexRef is unbound. + // No need to check for errors. + ihipDestroyTextureObject(texref->textureObject); + + hipResourceDesc resDesc = {}; + resDesc.resType = hipResourceTypeArray; + resDesc.res.array.array = const_cast(array); + + hipTextureDesc texDesc = hip::getTextureDesc(texref, readMode); + + hipResourceViewFormat format = hip::getResourceViewFormat(*desc); + hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(array, format); + + return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, &resViewDesc); +} + +hipError_t hipBindTextureToArray(textureReference* texref, + hipArray_const_t array, + const hipChannelFormatDesc* desc) { + HIP_INIT_API(hipBindTextureToArray, texref, array, desc); + + // TODO need compiler support to extract the read mode from textureReference. + HIP_RETURN(ihipBindTextureToArray(texref, array, desc, hipReadModeElementType)); +} + +hipError_t ihipBindTextureImpl(TlsData *tls, + int dim, + hipTextureReadMode readMode, + size_t* offset, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t size, + textureReference* texref) { + HIP_INIT_API(ihipBindTextureImpl, tls, dim, readMode, offset, devPtr, desc, size, texref); + + (void)dim; // Silence compiler warnings. + + HIP_RETURN(ihipBindTexture(offset, texref, devPtr, desc, size, readMode)); +} + +hipError_t ihipBindTextureToArrayImpl(TlsData *tls, + int dim, + hipTextureReadMode readMode, + hipArray_const_t array, + const hipChannelFormatDesc& desc, + textureReference* texref) { + // TODO overload operator<<(ostream&, hipChannelFormatDesc&). + HIP_INIT_API(ihipBindTextureToArrayImpl, tls, dim, readMode, array, &desc, texref); + + (void)dim; // Silence compiler warnings. + + HIP_RETURN(ihipBindTextureToArray(texref, array, &desc, readMode)); +} + +hipError_t ihipBindTextureToMipmappedArray(textureReference* texref, + hipMipmappedArray_const_t mipmappedArray, + const hipChannelFormatDesc* desc, + hipTextureReadMode readMode) { + if ((texref == nullptr) || + (mipmappedArray == nullptr) || + (desc == nullptr)) { + return hipErrorInvalidValue; + } + + // Any previous address or HIP array state associated with the texture reference is superseded by this function. + // Any memory previously bound to hTexRef is unbound. + // No need to check for errors. + ihipDestroyTextureObject(texref->textureObject); + + hipResourceDesc resDesc = {}; + resDesc.resType = hipResourceTypeMipmappedArray; + resDesc.res.mipmap.mipmap = const_cast(mipmappedArray); + + hipTextureDesc texDesc = hip::getTextureDesc(texref, readMode); + + hipResourceViewFormat format = hip::getResourceViewFormat(*desc); + hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(mipmappedArray, format); + + return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, &resViewDesc); +} + +hipError_t hipBindTextureToMipmappedArray(textureReference* texref, + hipMipmappedArray_const_t mipmappedArray, + const hipChannelFormatDesc* desc) { + HIP_INIT_API(hipBindTextureToMipmappedArray, texref, mipmappedArray, desc); + + // TODO need compiler support to extract the read mode from textureReference. + HIP_RETURN(ihipBindTextureToMipmappedArray(texref, mipmappedArray, desc, hipReadModeElementType)); +} + +hipError_t ihipBindTexture2DImpl(int dim, + hipTextureReadMode readMode, + size_t* offset, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t width, + size_t height, + textureReference* texref, + size_t pitch) { + HIP_INIT_API(ihipBindTexture2DImpl, dim, readMode, offset, devPtr, desc, width, height, texref, pitch); + + (void)dim; // Silence compiler warnings. + + HIP_RETURN(ihipBindTexture2D(offset, texref, devPtr, desc, width, height, pitch, readMode)); +} + +hipError_t ihipUnbindTextureImpl(const hipTextureObject_t& textureObject) { + // TODO overload operator<<(ostream&, hipTextureObject_t&). + HIP_INIT_API(ihipUnbindTextureImpl, &textureObject); + + HIP_RETURN(ihipDestroyTextureObject(textureObject)); +} + +hipError_t hipUnbindTexture(const textureReference* texref) { + HIP_INIT_API(hipUnbindTexture, texref); + + if (texref == nullptr) { + return hipErrorInvalidValue; + } + + HIP_RETURN(ihipDestroyTextureObject(texref->textureObject)); +} + +hipError_t hipBindTexture(size_t* offset, + textureReference* texref, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t size) { + HIP_INIT_API(hipBindTexture, offset, texref, devPtr, desc, size); + + // TODO need compiler support to extract the read mode from textureReference. + HIP_RETURN(ihipBindTexture(offset, texref, devPtr, desc, size, hipReadModeElementType)); +} + +hipError_t hipGetChannelDesc(hipChannelFormatDesc* desc, + hipArray_const_t array) { + HIP_INIT_API(hipGetChannelDesc, desc, array); + + if ((desc == nullptr) || + (array == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + // It is UB to call hipGetChannelDesc() on an array created via hipArrayCreate()/hipArray3DCreate(). + // This is due to hip not differentiating between runtime and driver types. + *desc = array->desc; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipGetTextureAlignmentOffset(size_t* offset, + const textureReference* texref) { + HIP_INIT_API(hipGetTextureAlignmentOffset, offset, texref); + + if ((offset == nullptr) || + (texref == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + // TODO enforce alignment on devPtr. *offset = 0; HIP_RETURN(hipSuccess); } -hipError_t hipGetTextureReference(const textureReference** tex, const void* symbol) { - HIP_INIT_API(NONE, tex, symbol); +hipError_t hipGetTextureReference(const textureReference** texref, const void* symbol) { + HIP_INIT_API(hipGetTextureReference, texref, symbol); assert(0 && "Unimplemented"); HIP_RETURN(hipErrorNotSupported); } -hipError_t hipTexRefSetFormat(textureReference* tex, hipArray_Format fmt, int NumPackedComponents) { - HIP_INIT_API(NONE, tex, fmt, NumPackedComponents); +hipError_t hipTexRefSetFormat(textureReference* texRef, + hipArray_Format fmt, + int NumPackedComponents) { + HIP_INIT_API(hipTexRefSetFormat, texRef, fmt, NumPackedComponents); - if (tex == nullptr) { - HIP_RETURN(hipErrorInvalidImage); - } - - tex->format = fmt; - tex->numChannels = NumPackedComponents; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetFlags(textureReference* tex, unsigned int flags) { - HIP_INIT_API(NONE, tex, flags); - - if (tex == nullptr) { - HIP_RETURN(hipErrorInvalidImage); - } - - tex->normalized = flags; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefSetFilterMode(textureReference* tex, hipTextureFilterMode fm) { - HIP_INIT_API(NONE, tex, fm); - - if (tex == nullptr) { - HIP_RETURN(hipErrorInvalidImage); - } - - tex->filterMode = fm; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefGetAddressMode(hipTextureAddressMode* am, textureReference tex, int dim) { - HIP_INIT_API(NONE, am, &tex, dim); - - if ((am == nullptr) || (dim >= 3)) { + if (texRef == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - *am = tex.addressMode[dim]; + texRef->format = fmt; + texRef->numChannels = NumPackedComponents; HIP_RETURN(hipSuccess); } -hipError_t hipTexRefSetAddressMode(textureReference* tex, int dim, hipTextureAddressMode am) { - HIP_INIT_API(NONE, tex, dim, am); +hipError_t hipTexRefSetFlags(textureReference* texRef, + unsigned int Flags) { + HIP_INIT_API(hipTexRefSetFlags, texRef, Flags); - if (tex == nullptr) { - HIP_RETURN(hipErrorInvalidImage); - } - - tex->addressMode[dim] = am; - - HIP_RETURN(hipSuccess); -} - -hipError_t hipTexRefGetArray(hipArray_t* array, textureReference tex) { - HIP_INIT_API(NONE, array, &tex); - - hip::TextureObject* texture = nullptr; - - if ((array == nullptr) || (*array == nullptr)) { - HIP_RETURN(hipErrorInvalidImage); - } - - texture = reinterpret_cast(tex.textureObject); - if(hipResourceTypeArray != texture->resDesc.resType){ + if (texRef == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - if (texture->resDesc.res.array.array == nullptr) { - HIP_RETURN(hipErrorInvalidValue); - } - - **array = *(texture->resDesc.res.array.array); + // TODO add textureReference::flags. + // Using textureReference::normalized for this purpose is OK for now, + // because calling hipTexRefGetFlags() on a textureRefence after hipBindTexture() is UB + // due to HIP not differentiating between runtime and driver api. + texRef->normalized = Flags; HIP_RETURN(hipSuccess); } -hipError_t hipTexRefSetArray(textureReference* tex, hipArray_const_t array, unsigned int flags) { - HIP_INIT_API(NONE, tex, array, flags); +hipError_t hipTexRefSetFilterMode(textureReference* texRef, + hipTextureFilterMode fm) { + HIP_INIT_API(hipTexRefSetFilterMode, texRef, fm); - size_t offset = 0; - cl_mem_object_type clType; - - if ((tex == nullptr) || (array == nullptr)) { - HIP_RETURN(hipErrorInvalidImage); + if (texRef == nullptr) { + HIP_RETURN(hipErrorInvalidValue); } - switch(array->textureType) { - case hipTextureType3D: - clType = CL_MEM_OBJECT_IMAGE3D; - break; - case hipTextureType2D: - clType = CL_MEM_OBJECT_IMAGE2D; - break; - case hipTextureType1D: - clType = CL_MEM_OBJECT_IMAGE1D; - break; - default: - HIP_RETURN(hipErrorInvalidValue); - } - HIP_RETURN(ihipBindTexture(clType, &offset, tex, array->data, array->desc, array->width, - array->height, array->depth)); -} - -hipError_t hipTexRefGetAddress(hipDeviceptr_t* dev_ptr, textureReference tex) { - HIP_INIT_API(NONE, dev_ptr, &tex); - - hip::TextureObject* texture = nullptr; - device::Memory* dev_mem = nullptr; - - texture = reinterpret_cast(tex.textureObject); - if ((texture == nullptr) || (texture->image == nullptr)) { - HIP_RETURN(hipErrorInvalidImage); - } - - dev_mem = texture->image->getDeviceMemory(*hip::getCurrentDevice()->devices()[0]); - if (dev_mem == nullptr) { - HIP_RETURN(hipErrorInvalidImage); - } - - *dev_ptr = reinterpret_cast(dev_mem->virtualAddress()); + texRef->filterMode = fm; HIP_RETURN(hipSuccess); } -hipError_t hipTexRefSetAddress(size_t* offset, textureReference* tex, hipDeviceptr_t devPtr, - size_t size) { - HIP_INIT_API(NONE, offset, tex, devPtr, size); +hipError_t hipTexRefGetAddressMode(hipTextureAddressMode* pam, + textureReference texRef, + int dim) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetAddressMode, pam, &texRef, dim); - if (tex == nullptr) { - HIP_RETURN(hipErrorInvalidImage); - } - - cl_image_format image_format; - getDrvChannelOrderAndType(tex->format, tex->numChannels, - &image_format.image_channel_order, &image_format.image_channel_data_type); - const amd::Image::Format imageFormat(image_format); - - HIP_RETURN(ihipBindTexture(CL_MEM_OBJECT_IMAGE1D, offset, tex, devPtr, tex->channelDesc, size / imageFormat.getElementSize(), 1, size)); -} - -hipError_t hipTexRefSetAddress2D(textureReference* tex, const HIP_ARRAY_DESCRIPTOR* desc, - hipDeviceptr_t devPtr, size_t pitch) { - HIP_INIT_API(NONE, tex, desc, devPtr, pitch); - - if (desc == nullptr) { + if (pam == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - size_t offset; - HIP_RETURN(ihipBindTexture(CL_MEM_OBJECT_IMAGE2D, &offset, tex, devPtr, tex->channelDesc, desc->Width, desc->Height, pitch)); + // Currently, the only valid value for dim are 0 and 1. + if ((dim != 0) || (dim != 1)) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pam = texRef.addressMode[dim]; + + HIP_RETURN(hipSuccess); } + +hipError_t hipTexRefSetAddressMode(textureReference* texRef, + int dim, + hipTextureAddressMode am) { + HIP_INIT_API(hipTexRefSetAddressMode, texRef, dim, am); + + if (texRef == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + if ((dim < 0) || (dim > 2)) { + HIP_RETURN(hipErrorInvalidValue); + } + + texRef->addressMode[dim] = am; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefGetArray(hipArray_t* pArray, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetArray, pArray, &texRef); + + if (pArray == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + hipResourceDesc resDesc = {}; + // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. + hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef.textureObject); + if (error != hipSuccess) { + return HIP_RETURN(error); + } + + switch (resDesc.resType) { + case hipResourceTypeLinear: + case hipResourceTypePitch2D: + case hipResourceTypeMipmappedArray: + HIP_RETURN(hipErrorInvalidValue); + case hipResourceTypeArray: + *pArray = resDesc.res.array.array; + break; + } + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefSetArray(textureReference* texRef, + hipArray_const_t array, + unsigned int flags) { + HIP_INIT_API(hipTexRefSetArray, texRef, array, flags); + + if ((texRef == nullptr) || + (array == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + if (flags != HIP_TRSA_OVERRIDE_FORMAT) { + HIP_RETURN(hipErrorInvalidValue); + } + + // Any previous address or HIP array state associated with the texture reference is superseded by this function. + // Any memory previously bound to hTexRef is unbound. + // No need to check for errors. + ihipDestroyTextureObject(texRef->textureObject); + + hipResourceDesc resDesc = {}; + resDesc.resType = hipResourceTypeArray; + resDesc.res.array.array = const_cast(array); + + // TODO need compiler support to extract the read mode from textureReference. + hipTextureDesc texDesc = hip::getTextureDesc(texRef, hipReadModeElementType); + + hipResourceViewFormat format = hip::getResourceViewFormat(hip::getChannelFormatDesc(texRef->numChannels, texRef->format)); + hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(array, format); + + HIP_RETURN(ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, &resViewDesc)); +} + +hipError_t hipTexRefGetAddress(hipDeviceptr_t* dptr, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetAddress, dptr, &texRef); + + if (dptr == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + hipResourceDesc resDesc = {}; + // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. + hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef.textureObject); + if (error != hipSuccess) { + return HIP_RETURN(error); + } + + switch (resDesc.resType) { + // Need to verify. + // If the texture reference is not bound to any device memory range, + // return hipErroInvalidValue. + case hipResourceTypeArray: + case hipResourceTypeMipmappedArray: + HIP_RETURN(hipErrorInvalidValue); + case hipResourceTypeLinear: + *dptr = resDesc.res.linear.devPtr; + break; + case hipResourceTypePitch2D: + *dptr = resDesc.res.pitch2D.devPtr; + break; + } + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefSetAddress(size_t* ByteOffset, + textureReference* texRef, + hipDeviceptr_t dptr, + size_t bytes) { + HIP_INIT_API(hipTexRefSetAddress, ByteOffset, texRef, dptr, bytes); + + if (texRef == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + // Any previous address or HIP array state associated with the texture reference is superseded by this function. + // Any memory previously bound to hTexRef is unbound. + // No need to check for errors. + ihipDestroyTextureObject(texRef->textureObject); + + // If the device memory pointer was returned from hipMemAlloc(), + // the offset is guaranteed to be 0 and NULL may be passed as the ByteOffset parameter. + // TODO enforce alignment on devPtr. + if (ByteOffset != nullptr) { + *ByteOffset = 0; + } + + hipResourceDesc resDesc = {}; + resDesc.resType = hipResourceTypeLinear; + resDesc.res.linear.devPtr = dptr; + resDesc.res.linear.desc = hip::getChannelFormatDesc(texRef->numChannels, texRef->format); + resDesc.res.linear.sizeInBytes = bytes; + + // TODO add textureReference::flags. + // Using textureReference::normalized for this purpose is OK for now, + // because calling hipTexRefGetFlags() on a textureRefence after hipBindTexture() + // due to HIP not differentiating between runtime and driver api. + hipTextureReadMode readMode = hip::getReadMode(texRef->normalized); + texRef->sRGB = hip::getSRGB(texRef->normalized); + texRef->normalized = hip::getNormalizedCoords(texRef->normalized); + hipTextureDesc texDesc = hip::getTextureDesc(texRef, readMode); + + HIP_RETURN(ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, nullptr)); +} + +hipError_t hipTexRefSetAddress2D(textureReference* texRef, + const HIP_ARRAY_DESCRIPTOR* desc, + hipDeviceptr_t dptr, + size_t Pitch) { + HIP_INIT_API(hipTexRefSetAddress2D, texRef, desc, dptr, Pitch); + + if ((texRef == nullptr) || + (desc == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + // Any previous address or HIP array state associated with the texture reference is superseded by this function. + // Any memory previously bound to hTexRef is unbound. + // No need to check for errors. + ihipDestroyTextureObject(texRef->textureObject); + + hipResourceDesc resDesc = {}; + resDesc.resType = hipResourceTypePitch2D; + resDesc.res.linear.devPtr = dptr; + resDesc.res.linear.desc = hip::getChannelFormatDesc(desc->NumChannels, desc->Format); // Need to verify. + resDesc.res.pitch2D.width = desc->Width; + resDesc.res.pitch2D.height = desc->Height; + resDesc.res.pitch2D.pitchInBytes = Pitch; + + // TODO add textureReference::flags. + // Using textureReference::normalized for this purpose is OK for now, + // because calling hipTexRefGetFlags() on a textureRefence after hipBindTexture() + // due to HIP not differentiating between runtime and driver api. + hipTextureReadMode readMode = hip::getReadMode(texRef->normalized); + texRef->sRGB = hip::getSRGB(texRef->normalized); + texRef->normalized = hip::getNormalizedCoords(texRef->normalized); + hipTextureDesc texDesc = hip::getTextureDesc(texRef, readMode); + + HIP_RETURN(ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, nullptr)); +} + +hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f) { + return {x, y, z, w, f}; +} + +hipError_t ihipBindTextureToMipmappedArrayImpl(TlsData *tls, + int dim, + hipTextureReadMode readMode, + hipMipmappedArray_const_t mipmappedArray, + const struct hipChannelFormatDesc& desc, + textureReference* texref) { + // TODO overload operator<<(ostream&, hipChannelFormatDesc&). + HIP_INIT_API(ihipBindTextureToMipmappedArrayImpl, tls, dim, readMode, mipmappedArray, &desc, texref); + + (void)dim; // Silence compiler warnings. + + HIP_RETURN(ihipBindTextureToMipmappedArray(texref, mipmappedArray, &desc, readMode)); +} + +hipError_t hipTexRefGetBorderColor(float* pBorderColor, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetBorderColor, pBorderColor, &texRef); + + if (pBorderColor == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + // TODO add textureReference::borderColor. + assert(false && "textureReference::borderColor is missing in header"); + // std::memcpy(pBorderColor, texRef.borderColor, sizeof(texRef.borderColor)); + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefGetFilterMode(hipTextureFilterMode* pfm, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetFilterMode, pfm, &texRef); + + if (pfm == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pfm = texRef.filterMode; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefGetFlags(unsigned int* pFlags, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetFlags, pFlags, &texRef); + + if (pFlags == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + // TODO add textureReference::flags. + // Using textureReference::normalized for this purpose is OK for now, + // because calling hipTexRefGetFlags() on a textureRefence after hipBindTexture() + // due to HIP not differentiating between runtime and driver api. + *pFlags = texRef.normalized; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefGetFormat(hipArray_Format* pFormat, + int* pNumChannels, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetFormat, pFormat, pNumChannels, &texRef); + + if ((pFormat == nullptr) || + (pNumChannels == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pFormat = texRef.format; + *pNumChannels = texRef.numChannels; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefGetMaxAnisotropy(int* pmaxAnsio, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetMaxAnisotropy, pmaxAnsio, &texRef); + + if (pmaxAnsio == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pmaxAnsio = texRef.maxAnisotropy; + + HIP_RETURN(hipErrorInvalidValue); +} + +hipError_t hipTexRefGetMipmapFilterMode(hipTextureFilterMode* pfm, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetMipmapFilterMode, pfm, &texRef); + + if (pfm == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pfm = texRef.mipmapFilterMode; + + HIP_RETURN(hipErrorInvalidValue); +} + +hipError_t hipTexRefGetMipmapLevelBias(float* pbias, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetMipmapLevelBias, pbias, &texRef); + + if (pbias == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pbias = texRef.mipmapLevelBias; + + HIP_RETURN(hipErrorInvalidValue); +} + +hipError_t hipTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, + float* pmaxMipmapLevelClamp, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetMipmapLevelClamp, pminMipmapLevelClamp, pmaxMipmapLevelClamp, &texRef); + + if ((pminMipmapLevelClamp == nullptr) || + (pmaxMipmapLevelClamp == nullptr)){ + HIP_RETURN(hipErrorInvalidValue); + } + + *pminMipmapLevelClamp = texRef.minMipmapLevelClamp; + *pmaxMipmapLevelClamp = texRef.maxMipmapLevelClamp; + + HIP_RETURN(hipErrorInvalidValue); +} + +hipError_t hipTexRefGetMipMappedArray(hipMipmappedArray_t* pArray, + textureReference texRef) { + // TODO overload operator<<(ostream&, textureReference&). + HIP_INIT_API(hipTexRefGetMipMappedArray, pArray, &texRef); + + if (pArray == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + hipResourceDesc resDesc = {}; + // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. + hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef.textureObject); + if (error != hipSuccess) { + return HIP_RETURN(error); + } + + switch (resDesc.resType) { + case hipResourceTypeLinear: + case hipResourceTypePitch2D: + case hipResourceTypeArray: + HIP_RETURN(hipErrorInvalidValue); + case hipResourceTypeMipmappedArray: + *pArray = resDesc.res.mipmap.mipmap; + break; + } + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefSetBorderColor(textureReference* texRef, + float* pBorderColor) { + HIP_INIT_API(hipTexRefSetBorderColor, texRef, pBorderColor); + + if ((texRef == nullptr) || + (pBorderColor == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + // TODO add textureReference::borderColor. + assert(false && "textureReference::borderColor is missing in header"); + // std::memcpy(texRef.borderColor, pBorderColor, sizeof(texRef.borderColor)); + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefSetMaxAnisotropy(textureReference* texRef, + unsigned int maxAniso) { + HIP_INIT_API(hipTexRefSetMaxAnisotropy, texRef, maxAniso); + + if (texRef == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + texRef->maxAnisotropy = maxAniso; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefSetMipmapFilterMode(textureReference* texRef, + hipTextureFilterMode fm) { + HIP_INIT_API(hipTexRefSetMipmapFilterMode, texRef, fm); + + if (texRef == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + texRef->mipmapFilterMode = fm; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefSetMipmapLevelBias(textureReference* texRef, + float bias) { + HIP_INIT_API(hipTexRefSetMipmapLevelBias, texRef, bias); + + if (texRef == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + texRef->mipmapLevelBias = bias; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefSetMipmapLevelClamp(textureReference* texRef, + float minMipMapLevelClamp, + float maxMipMapLevelClamp) { + HIP_INIT_API(hipTexRefSetMipmapLevelClamp, minMipMapLevelClamp, maxMipMapLevelClamp); + + if (texRef == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + texRef->minMipmapLevelClamp = minMipMapLevelClamp; + texRef->maxMipmapLevelClamp = maxMipMapLevelClamp; + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexRefSetMipmappedArray(textureReference* texRef, + hipMipmappedArray* mipmappedArray, + unsigned int Flags) { + HIP_INIT_API(hipTexRefSetMipmappedArray, texRef, mipmappedArray, Flags); + + if ((texRef == nullptr) || + (mipmappedArray == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + if (Flags != HIP_TRSA_OVERRIDE_FORMAT) { + HIP_RETURN(hipErrorInvalidValue); + } + + // Any previous address or HIP array state associated with the texture reference is superseded by this function. + // Any memory previously bound to hTexRef is unbound. + // No need to check for errors. + ihipDestroyTextureObject(texRef->textureObject); + + hipResourceDesc resDesc = {}; + resDesc.resType = hipResourceTypeMipmappedArray; + resDesc.res.mipmap.mipmap = mipmappedArray; + + // TODO need compiler support to extract the read mode from textureReference. + hipTextureDesc texDesc = hip::getTextureDesc(texRef, hipReadModeElementType); + + hipResourceViewFormat format = hip::getResourceViewFormat(hip::getChannelFormatDesc(texRef->numChannels, texRef->format)); + hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(mipmappedArray, format); + + HIP_RETURN(ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, &resViewDesc)); +} \ No newline at end of file From 9217b1ca707142ca7fdfabebb55d5db6906c0d61 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Sun, 23 Feb 2020 15:34:50 -0500 Subject: [PATCH 042/154] Reeanble texture tests for VDI Enable hipBindTex2DPitch and hipBindTexRef1DFetch Change-Id: I36a3252e25d1ed844967ec117e1f1951004cba11 --- tests/src/texture/hipBindTex2DPitch.cpp | 2 +- tests/src/texture/hipBindTexRef1DFetch.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/src/texture/hipBindTex2DPitch.cpp b/tests/src/texture/hipBindTex2DPitch.cpp index 8c57520c00..b01402c91d 100644 --- a/tests/src/texture/hipBindTex2DPitch.cpp +++ b/tests/src/texture/hipBindTex2DPitch.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /*HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp * TEST: %t * HIT_END */ diff --git a/tests/src/texture/hipBindTexRef1DFetch.cpp b/tests/src/texture/hipBindTexRef1DFetch.cpp index 650db5bc29..52a0d99ac1 100644 --- a/tests/src/texture/hipBindTexRef1DFetch.cpp +++ b/tests/src/texture/hipBindTexRef1DFetch.cpp @@ -22,7 +22,7 @@ THE SOFTWARE. /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp * TEST: %t * HIT_END */ From 73686f990d7a92ce69cce392fda5d6892bb8aebc Mon Sep 17 00:00:00 2001 From: Payam Date: Mon, 24 Feb 2020 17:37:21 -0500 Subject: [PATCH 043/154] updated hip-targets for hip-vdi Change-Id: Ib65e9011b25e368b8e03db1b4f14aa1ad076f4bc --- packaging/hip-targets-release.cmake | 32 +++++++++++++++---------- packaging/hip-targets.cmake | 37 +++++++++++++++++++---------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/packaging/hip-targets-release.cmake b/packaging/hip-targets-release.cmake index 36e9c6e9e5..c45af51bac 100644 --- a/packaging/hip-targets-release.cmake +++ b/packaging/hip-targets-release.cmake @@ -32,20 +32,28 @@ endif() # Import target "hip::hip_hcc" for configuration "Release" set_property(TARGET hip::hip_hcc APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) if(HIP_COMPILER STREQUAL "clang") -set_target_properties(hip::hip_hcc PROPERTIES - IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libhip_hcc.so" - IMPORTED_SONAME_RELEASE "libhip_hcc.so" - ) + if(HIP_RUNTIME STREQUAL "vdi") + set_target_properties(hip::amdhip64 PROPERTIES + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libamdhip64.so" + IMPORTED_SONAME_RELEASE "libamdhip64.so") + else() + set_target_properties(hip::hip_hcc PROPERTIES + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libhip_hcc.so" + IMPORTED_SONAME_RELEASE "libhip_hcc.so") + endif() else() -set_target_properties(hip::hip_hcc PROPERTIES - IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE "hcc::hccrt;hcc::hc_am" - IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libhip_hcc.so" - IMPORTED_SONAME_RELEASE "libhip_hcc.so" - ) + set_target_properties(hip::hip_hcc PROPERTIES + IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE "hcc::hccrt;hcc::hc_am" + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libhip_hcc.so" + IMPORTED_SONAME_RELEASE "libhip_hcc.so") endif() -list(APPEND _IMPORT_CHECK_TARGETS hip::hip_hcc ) -list(APPEND _IMPORT_CHECK_FILES_FOR_hip::hip_hcc "${_IMPORT_PREFIX}/lib/libhip_hcc.so" ) - +if(NOT HIP_RUNTIME STREQUAL "vdi") + list(APPEND _IMPORT_CHECK_TARGETS hip::hip_hcc ) + list(APPEND _IMPORT_CHECK_FILES_FOR_hip::hip_hcc "${_IMPORT_PREFIX}/lib/libhip_hcc.so" ) +else() + list(APPEND _IMPORT_CHECK_TARGETS hip::amdhip64) + list(APPEND _IMPORT_CHECK_FILES_FOR_hip::amdhip64 "${_IMPORT_PREFIX}/lib/libamdhip64.so" ) +endif() # Commands beyond this point should not need to know the version. set(CMAKE_IMPORT_FILE_VERSION) diff --git a/packaging/hip-targets.cmake b/packaging/hip-targets.cmake index 4d44579277..095c7cf62c 100644 --- a/packaging/hip-targets.cmake +++ b/packaging/hip-targets.cmake @@ -68,26 +68,39 @@ find_path(HSA_HEADER hsa/hsa.h if (HSA_HEADER-NOTFOUND) message (FATAL_ERROR "HSA header not found! ROCM_PATH environment not set") endif() -if(NOT HIP_RUNTIME STREQUAL "vdi") +if(HIP_RUNTIME STREQUAL "vdi") + # Create imported target hip::amdhip64 + add_library(hip::amdhip64 SHARED IMPORTED) + + set_target_properties(hip::amdhip64 PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include;${HSA_HEADER}" + INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include;${HSA_HEADER}" + ) +else() set_target_properties(hip::hip_hcc_static PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include;${HSA_HEADER}" INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include;${HSA_HEADER}") + + # Create imported target hip::hip_hcc + add_library(hip::hip_hcc SHARED IMPORTED) + + set_target_properties(hip::hip_hcc PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include;${HSA_HEADER}" + INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include;${HSA_HEADER}" + ) endif() -# Create imported target hip::hip_hcc -add_library(hip::hip_hcc SHARED IMPORTED) - -set_target_properties(hip::hip_hcc PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include;${HSA_HEADER}" - INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include;${HSA_HEADER}" -) - # Create imported target hip::host add_library(hip::host INTERFACE IMPORTED) -set_target_properties(hip::host PROPERTIES - INTERFACE_LINK_LIBRARIES "hip::hip_hcc" -) +if(HIP_RUNTIME STREQUAL "vdi") + set_target_properties(hip::host PROPERTIES + INTERFACE_LINK_LIBRARIES "hip::amdhip64") +else() + set_target_properties(hip::host PROPERTIES + INTERFACE_LINK_LIBRARIES "hip::hip_hcc") +endif() + # Create imported target hip::device add_library(hip::device INTERFACE IMPORTED) From 2203093159d7bb846cb3cf09e37260f622e53575 Mon Sep 17 00:00:00 2001 From: Christophe Paquot Date: Mon, 24 Feb 2020 13:28:08 -0800 Subject: [PATCH 044/154] Fixed a few multithreaded potential issues Also make D2H and H2D keep track of the chain of events when we need to use a different HostQueue. Change-Id: I1c5da6ea6104b37ad7aac00f0eb8ea9371e6ba1c --- vdi/hip_memory.cpp | 6 ++++++ vdi/hip_stream.cpp | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 63dd2c712e..58c27cb89d 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -102,6 +102,7 @@ hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKin amd::HostQueue* pQueue = &queue; if (queueDevice != dstMemory->getContext().devices()[0]) { pQueue = hip::getNullStream(dstMemory->getContext()); + waitList.push_back(queue.getLastQueuedCommand(true)); } command = new amd::WriteMemoryCommand(*pQueue, CL_COMMAND_WRITE_BUFFER, waitList, *dstMemory->asBuffer(), dOffset, sizeBytes, src); @@ -110,6 +111,7 @@ hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKin amd::HostQueue* pQueue = &queue; if (queueDevice != srcMemory->getContext().devices()[0]) { pQueue = hip::getNullStream(srcMemory->getContext()); + waitList.push_back(queue.getLastQueuedCommand(true)); } command = new amd::ReadMemoryCommand(*pQueue, CL_COMMAND_READ_BUFFER, waitList, *srcMemory->asBuffer(), sOffset, sizeBytes, dst); @@ -155,6 +157,10 @@ hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKin } command->release(); + if (waitList.size() > 0) { + waitList[0]->release(); + } + return hipSuccess; } diff --git a/vdi/hip_stream.cpp b/vdi/hip_stream.cpp index 7b32801aa8..55a975cc5c 100644 --- a/vdi/hip_stream.cpp +++ b/vdi/hip_stream.cpp @@ -224,7 +224,7 @@ hipError_t hipStreamQuery(hipStream_t stream) { hostQueue = reinterpret_cast(stream)->asHostQueue(); } - amd::Command* command = hostQueue->getLastQueuedCommand(false); + amd::Command* command = hostQueue->getLastQueuedCommand(true); if (command == nullptr) { HIP_RETURN(hipSuccess); } @@ -233,7 +233,9 @@ hipError_t hipStreamQuery(hipStream_t stream) { if (command->type() != 0) { event.notifyCmdQueue(); } - HIP_RETURN((command->status() == CL_COMPLETE) ? hipSuccess : hipErrorNotReady); + hipError_t status = (command->status() == CL_COMPLETE) ? hipSuccess : hipErrorNotReady; + command->release(); + HIP_RETURN(status); } hipError_t hipStreamAddCallback(hipStream_t stream, hipStreamCallback_t callback, void* userData, From a8206e2e8f246177648b3c026f44970b879563d0 Mon Sep 17 00:00:00 2001 From: Payam Date: Tue, 25 Feb 2020 02:10:05 -0500 Subject: [PATCH 045/154] changed variable name to VDI Change-Id: I9ab338b50714b0df2db1fa485134e1dad80b68c0 --- packaging/hip-targets-release.cmake | 6 +++--- packaging/hip-targets.cmake | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packaging/hip-targets-release.cmake b/packaging/hip-targets-release.cmake index c45af51bac..942f4f3be7 100644 --- a/packaging/hip-targets-release.cmake +++ b/packaging/hip-targets-release.cmake @@ -10,7 +10,7 @@ set(CMAKE_IMPORT_FILE_VERSION 1) get_filename_component(_DIR "${CMAKE_CURRENT_LIST_DIR}" REALPATH) get_filename_component(_IMPORT_PREFIX "${_DIR}/../../../" REALPATH) -if(NOT HIP_RUNTIME STREQUAL "vdi") +if(NOT HIP_RUNTIME MATCHES "VDI") # Import target "hip::hip_hcc_static" for configuration "Release" set_property(TARGET hip::hip_hcc_static APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) if(HIP_COMPILER STREQUAL "clang") @@ -32,7 +32,7 @@ endif() # Import target "hip::hip_hcc" for configuration "Release" set_property(TARGET hip::hip_hcc APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) if(HIP_COMPILER STREQUAL "clang") - if(HIP_RUNTIME STREQUAL "vdi") + if(HIP_RUNTIME MATCHES "VDI") set_target_properties(hip::amdhip64 PROPERTIES IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libamdhip64.so" IMPORTED_SONAME_RELEASE "libamdhip64.so") @@ -48,7 +48,7 @@ else() IMPORTED_SONAME_RELEASE "libhip_hcc.so") endif() -if(NOT HIP_RUNTIME STREQUAL "vdi") +if(NOT HIP_RUNTIME MATCHES "VDI") list(APPEND _IMPORT_CHECK_TARGETS hip::hip_hcc ) list(APPEND _IMPORT_CHECK_FILES_FOR_hip::hip_hcc "${_IMPORT_PREFIX}/lib/libhip_hcc.so" ) else() diff --git a/packaging/hip-targets.cmake b/packaging/hip-targets.cmake index 095c7cf62c..90a76bef0f 100644 --- a/packaging/hip-targets.cmake +++ b/packaging/hip-targets.cmake @@ -16,7 +16,7 @@ set(CMAKE_IMPORT_FILE_VERSION 1) set(_targetsDefined) set(_targetsNotDefined) set(_expectedTargets) -if(HIP_RUNTIME STREQUAL "vdi") +if(HIP_RUNTIME MATCHES "VDI") foreach(_expectedTarget hip::hip_hcc hip::host hip::device) else() foreach(_expectedTarget hip:hip_hcc_static hip::hip_hcc hip::host hip::device) @@ -52,7 +52,7 @@ get_filename_component(_DIR "${CMAKE_CURRENT_LIST_DIR}" REALPATH) get_filename_component(_IMPORT_PREFIX "${_DIR}/../../../" REALPATH) # Create imported target hip::hip_hcc_static -if( NOT HIP_RUNTIME STREQUAL "vdi") +if( NOT HIP_RUNTIME MATCHES "VDI") add_library(hip::hip_hcc_static STATIC IMPORTED) endif() @@ -68,7 +68,7 @@ find_path(HSA_HEADER hsa/hsa.h if (HSA_HEADER-NOTFOUND) message (FATAL_ERROR "HSA header not found! ROCM_PATH environment not set") endif() -if(HIP_RUNTIME STREQUAL "vdi") +if(HIP_RUNTIME MATCHES "VDI") # Create imported target hip::amdhip64 add_library(hip::amdhip64 SHARED IMPORTED) @@ -93,7 +93,7 @@ endif() # Create imported target hip::host add_library(hip::host INTERFACE IMPORTED) -if(HIP_RUNTIME STREQUAL "vdi") +if(HIP_RUNTIME MATCHES "VDI") set_target_properties(hip::host PROPERTIES INTERFACE_LINK_LIBRARIES "hip::amdhip64") else() From 631fd65be41b9d70d7f71ea985fc9da9eb9755ee Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Tue, 25 Feb 2020 00:09:49 -0800 Subject: [PATCH 046/154] Use the context variant of getNullStream Do not create a new queue to call finish in hipFree if none was created earlier elsewhere. Change-Id: I87bb191e6b186ddbe607ab29d11e3ae5bc2ac8e6 --- vdi/hip_context.cpp | 1 + vdi/hip_memory.cpp | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/vdi/hip_context.cpp b/vdi/hip_context.cpp index ebc6f88ee3..f28ca48239 100644 --- a/vdi/hip_context.cpp +++ b/vdi/hip_context.cpp @@ -103,6 +103,7 @@ amd::HostQueue* getNullStream(Device& dev) { } return stream->second; } + amd::HostQueue* getNullStream(amd::Context& ctx) { for (auto& it : g_nullStreams) { if (it.first->asContext() == &ctx) { diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 58c27cb89d..77d48a4368 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -224,8 +224,11 @@ hipError_t hipFree(void* ptr) { } if (amd::SvmBuffer::malloced(ptr)) { hip::syncStreams(); - for (size_t i=0; ifinish(); + for (auto& dev : g_devices) { + amd::HostQueue* queue = hip::getNullStream(*dev->asContext()); + if (queue != nullptr) { + queue->finish(); + } } amd::SvmBuffer::free(*hip::getCurrentDevice()->asContext(), ptr); HIP_RETURN(hipSuccess); From ccc73a9fb4bfa41330be710fb77b06b41994bcea Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 25 Feb 2020 13:50:34 -0500 Subject: [PATCH 047/154] hipBindTexture() should handle nullptr offset. If a user passes a ptr that was allocated by hipMalloc(), the offset is guaranteed to be 0 and NULL may be passed as the offset parameter. We shouldn't return an error in this case. Change-Id: I4a8d645121e5a17d5e2861a0629356a3599de9ee --- vdi/hip_texture.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 634c83557c..bab61340ca 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -388,8 +388,7 @@ hipError_t ihipBindTexture(size_t* offset, const hipChannelFormatDesc* desc, size_t size, hipTextureReadMode readMode) { - if ((offset == nullptr) || - (texref == nullptr) || + if ((texref == nullptr) || (devPtr == nullptr) || (desc == nullptr)) { return hipErrorInvalidValue; @@ -426,8 +425,7 @@ hipError_t ihipBindTexture2D(size_t* offset, size_t height, size_t pitch, hipTextureReadMode readMode) { - if ((offset == nullptr) || - (texref == nullptr) || + if ((texref == nullptr) || (devPtr == nullptr) || (desc == nullptr)) { return hipErrorInvalidValue; From 657fb46b456605e212bc6e2b18506595ec8aad2d Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 25 Feb 2020 13:54:43 -0500 Subject: [PATCH 048/154] Bump c++ version to 14. This is to match our p4 build. Change-Id: I084b07968ced98cca216146a0228cc36e9f56ab3 --- vdi/CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vdi/CMakeLists.txt b/vdi/CMakeLists.txt index 0140408875..e132323a50 100644 --- a/vdi/CMakeLists.txt +++ b/vdi/CMakeLists.txt @@ -1,6 +1,5 @@ #project("hip") cmake_minimum_required(VERSION 3.5.1) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--no-keep-memory -Wl,-Bsymbolic -Wl,--unresolved-symbols=report-all -Wl,--version-script=${CMAKE_CURRENT_LIST_DIR}/hip_hcc.map.in") @@ -133,6 +132,13 @@ add_library(amdhip64 SHARED fixme.cpp ) +set_target_properties( + amdhip64 PROPERTIES + CXX_STANDARD 14 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF +) + set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) include(${LIBVDI_STATIC_DIR}/amdvdi_staticTargets.cmake) From 224d41c2f083a3f53b246aa7e163844428e6d9c0 Mon Sep 17 00:00:00 2001 From: Payam Date: Tue, 25 Feb 2020 16:16:06 -0500 Subject: [PATCH 049/154] removed excutable permission from source files Change-Id: Iae8639a96c55a098e28de41c5a3f38a07acbe25c --- vdi/hip_internal.hpp | 0 vdi/hip_module.cpp | 0 vdi/hip_platform.cpp | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 vdi/hip_internal.hpp mode change 100755 => 100644 vdi/hip_module.cpp mode change 100755 => 100644 vdi/hip_platform.cpp diff --git a/vdi/hip_internal.hpp b/vdi/hip_internal.hpp old mode 100755 new mode 100644 diff --git a/vdi/hip_module.cpp b/vdi/hip_module.cpp old mode 100755 new mode 100644 diff --git a/vdi/hip_platform.cpp b/vdi/hip_platform.cpp old mode 100755 new mode 100644 From f4401ff2b535f34f8e7e7c99e6e8b8602de221f1 Mon Sep 17 00:00:00 2001 From: Payam Date: Tue, 25 Feb 2020 13:44:51 -0500 Subject: [PATCH 050/154] removing 3 tests temporarily to enable hiptests Change-Id: I8c45791aaf88628850462aff90979c147cb85f64 --- tests/src/gcc/hipMalloc.c | 4 ++-- tests/src/hiprtc/saxpy.cpp | 2 +- tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/src/gcc/hipMalloc.c b/tests/src/gcc/hipMalloc.c index 63de6b2ad7..ebf163de28 100644 --- a/tests/src/gcc/hipMalloc.c +++ b/tests/src/gcc/hipMalloc.c @@ -19,9 +19,9 @@ /* HIT_START * BUILD_CMD: hipMalloc %cc -D__HIP_PLATFORM_NVCC__ -I%hip-path/include -I/usr/local/cuda/include %S/%s -o %T/hipMalloc_nv -L/usr/local/cuda/lib64 -lcudart EXCLUDE_HIP_PLATFORM hcc vdi - * BUILD_CMD: hipMalloc %cc -D__HIP_PLATFORM_HCC__ -I%hip-path/include %S/%s -Wl,--rpath=%hip-path/lib %hip-path/lib/libhip_hcc.so -o %T/hipMalloc_hcc EXCLUDE_HIP_PLATFORM nvcc + * BUILD_CMD: hipMalloc %cc -D__HIP_PLATFORM_HCC__ -I%hip-path/include %S/%s -Wl,--rpath=%hip-path/lib %hip-path/lib/libhip_hcc.so -o %T/hipMalloc_hcc EXCLUDE_HIP_PLATFORM nvcc vdi * TEST: hipMalloc_nv EXCLUDE_HIP_PLATFORM hcc vdi - * TEST: hipMalloc_hcc EXCLUDE_HIP_PLATFORM nvcc + * TEST: hipMalloc_hcc EXCLUDE_HIP_PLATFORM nvcc vdi * HIT_END */ diff --git a/tests/src/hiprtc/saxpy.cpp b/tests/src/hiprtc/saxpy.cpp index d063578757..a08c1c2399 100755 --- a/tests/src/hiprtc/saxpy.cpp +++ b/tests/src/hiprtc/saxpy.cpp @@ -20,7 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp LINK_OPTIONS hiprtc EXCLUDE_HIP_PLATFORM nvcc + * BUILD: %t %s ../test_common.cpp LINK_OPTIONS hiprtc EXCLUDE_HIP_PLATFORM nvcc vdi * TEST: %t * HIT_END */ diff --git a/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp b/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp index 9ae5883608..3903acd125 100644 --- a/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp +++ b/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi * TEST: %t * HIT_END */ From 69404d8e78f9eeebd11e28e85b47fb9e1bf90075 Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Wed, 26 Feb 2020 02:23:43 -0800 Subject: [PATCH 051/154] Fix hipcc for extra -mllvm option (#1885) --- bin/hipcc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/hipcc b/bin/hipcc index 41f11a36b4..22923e5ec4 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -729,7 +729,9 @@ if ($HIP_PLATFORM eq "clang") { } if ($optArg ne "-O0") { $HIPCXXFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; - $HIPLDFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; + if ($needLDFLAGS and not $needCXXFLAGS) { + $HIPLDFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; + } } $HIP_DEVLIB_FLAGS = " --hip-device-lib-path=$DEVICE_LIB_PATH"; $HIPCXXFLAGS .= " $HIP_DEVLIB_FLAGS"; From b7dd073d9367748c48a474265d2d79cbf5d9cfe0 Mon Sep 17 00:00:00 2001 From: Nick Curtis Date: Wed, 26 Feb 2020 04:23:56 -0600 Subject: [PATCH 052/154] fix long shuffle implementations for windows (#1895) Fixes for SWDEV-223694 --- include/hip/hcc_detail/device_functions.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/hip/hcc_detail/device_functions.h b/include/hip/hcc_detail/device_functions.h index 46ed53ff87..7096841da8 100644 --- a/include/hip/hcc_detail/device_functions.h +++ b/include/hip/hcc_detail/device_functions.h @@ -323,6 +323,7 @@ __device__ inline long __shfl(long var, int src_lane, int width = warpSize) { + #ifndef _MSC_VER static_assert(sizeof(long) == 2 * sizeof(int), ""); static_assert(sizeof(long) == sizeof(uint64_t), ""); @@ -333,6 +334,10 @@ long __shfl(long var, int src_lane, int width = warpSize) uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); return tmp1; + #else + static_assert(sizeof(long) == sizeof(int), ""); + return static_cast(__shfl(static_cast(var), src_lane, width)); + #endif } __device__ inline @@ -390,6 +395,7 @@ __device__ inline long __shfl_up(long var, unsigned int lane_delta, int width = warpSize) { + #ifndef _MSC_VER static_assert(sizeof(long) == 2 * sizeof(int), ""); static_assert(sizeof(long) == sizeof(uint64_t), ""); @@ -400,6 +406,10 @@ long __shfl_up(long var, unsigned int lane_delta, int width = warpSize) uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); return tmp1; + #else + static_assert(sizeof(long) == sizeof(int), ""); + return static_cast(__shfl_up(static_cast(var), lane_delta, width)); + #endif } __device__ inline @@ -455,6 +465,7 @@ __device__ inline long __shfl_down(long var, unsigned int lane_delta, int width = warpSize) { + #ifndef _MSC_VER static_assert(sizeof(long) == 2 * sizeof(int), ""); static_assert(sizeof(long) == sizeof(uint64_t), ""); @@ -465,6 +476,10 @@ long __shfl_down(long var, unsigned int lane_delta, int width = warpSize) uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); return tmp1; + #else + static_assert(sizeof(long) == sizeof(int), ""); + return static_cast(__shfl_down(static_cast(var), lane_delta, width)); + #endif } __device__ inline @@ -520,6 +535,7 @@ __device__ inline long __shfl_xor(long var, int lane_mask, int width = warpSize) { + #ifndef _MSC_VER static_assert(sizeof(long) == 2 * sizeof(int), ""); static_assert(sizeof(long) == sizeof(uint64_t), ""); @@ -530,6 +546,10 @@ long __shfl_xor(long var, int lane_mask, int width = warpSize) uint64_t tmp0 = (static_cast(tmp[1]) << 32ull) | static_cast(tmp[0]); long tmp1; __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0)); return tmp1; + #else + static_assert(sizeof(long) == sizeof(int), ""); + return static_cast(__shfl_down(static_cast(var), lane_delta, width)); + #endif } __device__ inline From fa6ed89cff5afa2a40efb58f74ade370ffa19c56 Mon Sep 17 00:00:00 2001 From: Christophe Paquot Date: Wed, 26 Feb 2020 08:41:18 -0800 Subject: [PATCH 053/154] Blocking and default streams' sync: Add hip::syncStreams(dev) to sync blocking streams on a given device. hip::syncStreams(void) should only sync streams on the current device. Change-Id: Ib6b0735215fa0ed12c646ebd029e9763ee3712ce --- vdi/hip_internal.hpp | 3 ++- vdi/hip_memory.cpp | 22 ++++++++++++++-------- vdi/hip_stream.cpp | 10 ++++++++-- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/vdi/hip_internal.hpp b/vdi/hip_internal.hpp index ad05e51b47..86012438fa 100644 --- a/vdi/hip_internal.hpp +++ b/vdi/hip_internal.hpp @@ -111,8 +111,9 @@ namespace hip { /// Get default stream of the thread extern amd::HostQueue* getNullStream(); /// Sync Blocking streams on the current device - /// TODO: It currently syncs all Blocking streams on all devices extern void syncStreams(); + /// Sync blocking streams on the given device + extern void syncStreams(int devId); struct Function { diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 77d48a4368..6d68cfd9f5 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -65,8 +65,10 @@ hipError_t ihipMalloc(void** ptr, size_t sizeBytes, unsigned int flags) *ptr = amd::SvmBuffer::malloc(*amdContext, flags, sizeBytes, amdContext->devices()[0]->info().memBaseAddrAlign_); if (*ptr == nullptr) { - hip::syncStreams(); - hip::getNullStream()->finish(); + for (auto& dev : g_devices) { + hip::getNullStream(*dev->asContext())->finish(); + hip::syncStreams(dev->deviceId()); + } *ptr = amd::SvmBuffer::malloc(*amdContext, flags, sizeBytes, amdContext->devices()[0]->info().memBaseAddrAlign_); if (*ptr == nullptr) { @@ -223,12 +225,12 @@ hipError_t hipFree(void* ptr) { HIP_RETURN(hipSuccess); } if (amd::SvmBuffer::malloced(ptr)) { - hip::syncStreams(); for (auto& dev : g_devices) { amd::HostQueue* queue = hip::getNullStream(*dev->asContext()); if (queue != nullptr) { queue->finish(); } + hip::syncStreams(dev->deviceId()); } amd::SvmBuffer::free(*hip::getCurrentDevice()->asContext(), ptr); HIP_RETURN(hipSuccess); @@ -676,9 +678,15 @@ hipError_t hipHostRegister(void* hostPtr, size_t sizeBytes, unsigned int flags) hipError_t hipHostUnregister(void* hostPtr) { HIP_INIT_API(hipHostUnregister, hostPtr); + for (auto& dev : g_devices) { + amd::HostQueue* queue = hip::getNullStream(*dev->asContext()); + if (queue != nullptr) { + queue->finish(); + } + hip::syncStreams(dev->deviceId()); + } + if (amd::SvmBuffer::malloced(hostPtr)) { - hip::syncStreams(); - hip::getNullStream()->finish(); amd::SvmBuffer::free(*hip::host_device->asContext(), hostPtr); HIP_RETURN(hipSuccess); } else { @@ -686,8 +694,6 @@ hipError_t hipHostUnregister(void* hostPtr) { amd::Memory* mem = getMemoryObject(hostPtr, offset); if(mem) { - hip::syncStreams(); - hip::getNullStream()->finish(); for (const auto& device: hip::getCurrentDevice()->devices()) { const device::Memory* devMem = mem->getDeviceMemory(*device); amd::MemObjMap::RemoveMemObj(reinterpret_cast(devMem->virtualAddress())); @@ -2033,4 +2039,4 @@ hipError_t hipMemcpyHtoAAsync(hipArray* dstArray, HIP_INIT_API(hipMemcpyHtoAAsync, dstArray, dstOffset, srcHost, ByteCount, stream); HIP_RETURN(ihipMemcpyHtoA(srcHost, dstArray, {0, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, 0, 0, stream, true)); -} \ No newline at end of file +} diff --git a/vdi/hip_stream.cpp b/vdi/hip_stream.cpp index 55a975cc5c..53311ae4ff 100644 --- a/vdi/hip_stream.cpp +++ b/vdi/hip_stream.cpp @@ -42,14 +42,20 @@ class StreamCallback { namespace hip { -void syncStreams() { +void syncStreams(int devId) { amd::ScopedLock lock(streamSetLock); for (const auto& it : streamSet) { - it->finish(); + if (it->device->deviceId() == devId) { + it->finish(); + } } } +void syncStreams() { + syncStreams(getCurrentDevice()->deviceId()); +} + Stream::Stream(hip::Device* dev, amd::CommandQueue::Priority p, unsigned int f) : queue(nullptr), device(dev), priority(p), flags(f) {} From 71e1f87f7e392e24756538c14a95e0e652b2082c Mon Sep 17 00:00:00 2001 From: Maneesh Gupta Date: Thu, 27 Feb 2020 16:18:31 +0530 Subject: [PATCH 054/154] bump version to 3.2 (#1898) - Bump version to 3.2 - [ci] Enable tests on ROCm 3.1 --- Jenkinsfile | 8 ++++---- bin/hipconfig | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 734e875e03..e38f7824d2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -295,13 +295,13 @@ def docker_upload_dockerhub( String local_org, String image_name, String remote_ String build_config = 'Release' String job_name = env.JOB_NAME.toLowerCase( ) -// The following launches 3 builds in parallel: rocm-head, rocm-3.0.x and cuda-10.x -parallel rocm_3_0: +// The following launches 3 builds in parallel: rocm-head, rocm-3.1.x and cuda-10.x +parallel rocm_3_1: { node('hip-rocm') { - String hcc_ver = 'rocm-3.0.x' - String from_image = 'ci_test_nodes/rocm-3.0.x/ubuntu-16.04:latest' + String hcc_ver = 'rocm-3.1.x' + String from_image = 'ci_test_nodes/rocm-3.1.x/ubuntu-16.04:latest' String inside_args = '--device=/dev/kfd --device=/dev/dri --group-add=video' // Checkout source code, dependencies and version files diff --git a/bin/hipconfig b/bin/hipconfig index a73e8af8b9..c56b56ecd8 100755 --- a/bin/hipconfig +++ b/bin/hipconfig @@ -1,7 +1,7 @@ #!/usr/bin/perl -w $HIP_BASE_VERSION_MAJOR = "3"; -$HIP_BASE_VERSION_MINOR = "1"; +$HIP_BASE_VERSION_MINOR = "2"; # Need perl > 5.10 to use logic-defined or use 5.006; use v5.10.1; From d830dad3beac21f661f30ffc7e91e6f74f076496 Mon Sep 17 00:00:00 2001 From: Alex Voicu Date: Thu, 27 Feb 2020 12:51:12 +0200 Subject: [PATCH 055/154] Address post-staging issues in #1809 (#1894) Fixes SWDEV-223910 and SWDEV-223663 --- src/hip_memory.cpp | 95 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/src/hip_memory.cpp b/src/hip_memory.cpp index 1bcf10f982..8159f22a97 100644 --- a/src/hip_memory.cpp +++ b/src/hip_memory.cpp @@ -139,6 +139,8 @@ namespace { return r; }()}; + constexpr std::uint32_t is_cpu_owned{UINT32_MAX}; + inline hsa_amd_pointer_info_t info(const void* p) { @@ -148,14 +150,14 @@ namespace { const_cast(p), &r, nullptr, nullptr, nullptr), __FILE__, __func__, __LINE__); - if (is_large_BAR) r.size = UINT32_MAX; - else if (type(r.agentOwner) == HSA_DEVICE_TYPE_CPU) r.size = INT32_MAX; + if (type(r.agentOwner) == HSA_DEVICE_TYPE_CPU) r.size = is_cpu_owned; return r; } - constexpr size_t staging_sz{4 * 1024 * 1024}; // 2 Pages. - constexpr size_t max_std_memcpy_sz{8 * 1024}; // 8 KiB. + constexpr size_t staging_sz{4 * 1024 * 1024}; // 2 Pages. + constexpr size_t max_h2d_std_memcpy_sz{8 * 1024}; // 8 KiB. + constexpr size_t max_d2h_std_memcpy_sz{64}; // 1 cacheline. thread_local const std::unique_ptr staging_buffer{ []() { @@ -203,7 +205,7 @@ namespace { inline void do_copy(void* __restrict dst, const void* __restrict src, size_t n, - hsa_agent_t da, hsa_agent_t sa) { + hsa_agent_t da, hsa_agent_t sa) { hsa_signal_silent_store_relaxed(copy_signal, 1); throwing_result_check( hsa_amd_memory_async_copy(dst, da, src, sa, n, 0, nullptr, copy_signal), @@ -224,14 +226,20 @@ void do_std_memcpy( inline void d2h_copy(void* __restrict dst, const void* __restrict src, size_t n, hsa_amd_pointer_info_t si) { - if (si.size == INT32_MAX) return do_std_memcpy(dst, src, n); - if (si.size == UINT32_MAX && n <= max_std_memcpy_sz) { + const auto di{info(dst)}; + const auto is_locked{di.type == HSA_EXT_POINTER_TYPE_LOCKED}; + + if (!is_locked && si.size == is_cpu_owned) { return do_std_memcpy(dst, src, n); } - - const auto di{info(dst)}; - - if (di.type == HSA_EXT_POINTER_TYPE_LOCKED) { + if (!is_locked && is_large_BAR && n <= max_d2h_std_memcpy_sz) { + return do_std_memcpy(dst, src, n); + } + if (di.type == HSA_EXT_POINTER_TYPE_HSA) { + return do_copy(dst, src, n, si.agentOwner, si.agentOwner); + } + + if (is_locked) { dst = static_cast(di.agentBaseAddress) + (static_cast(dst) - static_cast(di.hostBaseAddress)); @@ -247,7 +255,7 @@ void d2h_copy(void* __restrict dst, const void* __restrict src, size_t n, throwing_result_check(hsa_amd_memory_lock(dst, n, &si.agentOwner, 1, const_cast(&dst)), - __FILE__, __func__, __LINE__); + __FILE__, __func__, __LINE__); do_copy(dst, src, n, si.agentOwner, si.agentOwner); } @@ -256,14 +264,20 @@ void d2h_copy(void* __restrict dst, const void* __restrict src, size_t n, inline void h2d_copy(void* __restrict dst, const void* __restrict src, size_t n, hsa_amd_pointer_info_t di) { - if (di.size == INT32_MAX) return do_std_memcpy(dst, src, n); - if (di.size == UINT32_MAX && n <= max_std_memcpy_sz) { + const auto si{info(const_cast(src))}; + const auto is_locked{si.type == HSA_EXT_POINTER_TYPE_LOCKED}; + + if (!is_locked && di.size == is_cpu_owned) { return do_std_memcpy(dst, src, n); } + if (!is_locked && is_large_BAR && n <= max_h2d_std_memcpy_sz) { + return do_std_memcpy(dst, src, n); + } + if (si.type == HSA_EXT_POINTER_TYPE_HSA) { + return do_copy(dst, src, n, di.agentOwner, di.agentOwner); + } - const auto si{info(const_cast(src))}; - - if (si.type == HSA_EXT_POINTER_TYPE_LOCKED) { + if (is_locked) { src = static_cast(si.agentBaseAddress) + (static_cast(src) - static_cast(si.hostBaseAddress)); @@ -280,7 +294,7 @@ void h2d_copy(void* __restrict dst, const void* __restrict src, size_t n, throwing_result_check(hsa_amd_memory_lock(const_cast(src), n, &di.agentOwner, 1, const_cast(&src)), - __FILE__, __func__, __LINE__); + __FILE__, __func__, __LINE__); do_copy(dst, src, n, di.agentOwner, di.agentOwner); } @@ -289,36 +303,23 @@ void h2d_copy(void* __restrict dst, const void* __restrict src, size_t n, inline void generic_copy(void* __restrict dst, const void* __restrict src, size_t n, hsa_amd_pointer_info_t di, hsa_amd_pointer_info_t si) { - if (di.size == INT32_MAX && si.size == INT32_MAX) { - return do_std_memcpy(dst, src, n); - } - if (di.size == UINT32_MAX && si.size == UINT32_MAX && - n <= max_std_memcpy_sz) { + if (di.size == is_cpu_owned && si.size == is_cpu_owned) { return do_std_memcpy(dst, src, n); } + if (di.size == is_cpu_owned) return d2h_copy(dst, src, n, si); + if (si.size == is_cpu_owned) return h2d_copy(dst, src, n, di); - switch (type(si.agentOwner)) { - case HSA_DEVICE_TYPE_GPU: - if (type(di.agentOwner) == HSA_DEVICE_TYPE_GPU) { - throwing_result_check( - hsa_amd_agents_allow_access( - 1u, &si.agentOwner, nullptr, di.agentBaseAddress), - __FILE__, __func__, __LINE__); - return do_copy(dst, src, n, di.agentOwner, si.agentOwner); - } - return d2h_copy(dst, src, n, si); - case HSA_DEVICE_TYPE_CPU: - if (type(di.agentOwner) == HSA_DEVICE_TYPE_CPU) { - return do_std_memcpy(dst, src, n); - } - return h2d_copy(dst, src, n, di); - default: throw std::runtime_error{"Unsupported copy type."}; - } + throwing_result_check(hsa_amd_agents_allow_access(1u, &si.agentOwner, + nullptr, + di.agentBaseAddress), + __FILE__, __func__, __LINE__); + + return do_copy(dst, src, n, di.agentOwner, si.agentOwner); } inline void memcpy_impl(void* __restrict dst, const void* __restrict src, size_t n, - hipMemcpyKind k) noexcept { + hipMemcpyKind k) { switch (k) { case hipMemcpyHostToHost: std::memcpy(dst, src, n); break; case hipMemcpyHostToDevice: return h2d_copy(dst, src, n, info(dst)); @@ -326,10 +327,10 @@ void memcpy_impl(void* __restrict dst, const void* __restrict src, size_t n, case hipMemcpyDeviceToDevice: { const auto di{info(dst)}; const auto si{info(src)}; - throwing_result_check( - hsa_amd_agents_allow_access( - 1u, &si.agentOwner, nullptr, di.agentBaseAddress), - __FILE__, __func__, __LINE__); + throwing_result_check(hsa_amd_agents_allow_access(1u, &si.agentOwner, + nullptr, + di.agentBaseAddress), + __FILE__, __func__, __LINE__); return do_copy(dst, src, n, di.agentOwner, si.agentOwner); } default: return generic_copy(dst, src, n, info(dst), info(src)); @@ -1271,7 +1272,7 @@ hipError_t hipMemcpyToSymbolAsync(void* dst, const void* src, size_t count, if (dst == nullptr) { return ihipLogStatus(hipErrorInvalidSymbol); } - + if (kind == hipMemcpyDeviceToHost || kind == hipMemcpyHostToHost) { return ihipLogStatus(hipErrorInvalidMemcpyDirection); } else if (kind == hipMemcpyDeviceToDevice) { @@ -1303,7 +1304,7 @@ hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* src, size_t count, if (src == nullptr || dst == nullptr) { return ihipLogStatus(hipErrorInvalidSymbol); } - + if (kind == hipMemcpyHostToDevice || kind == hipMemcpyHostToHost) { return ihipLogStatus(hipErrorInvalidMemcpyDirection); } else if (kind == hipMemcpyDeviceToDevice) { From 6f2a872d87e0a542b62db2c49d4acca8cc76a351 Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Thu, 27 Feb 2020 10:52:13 -0800 Subject: [PATCH 056/154] Add -Wno-ignored-attributes flag to mute warnings Change-Id: Ib9ad75554c0ddfe025e7d500f057830f7ad2ef5b --- vdi/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vdi/CMakeLists.txt b/vdi/CMakeLists.txt index e132323a50..92332cb802 100644 --- a/vdi/CMakeLists.txt +++ b/vdi/CMakeLists.txt @@ -18,6 +18,10 @@ if(CMAKE_BUILD_TYPE MATCHES "^Debug$") add_definitions(-DDEBUG) endif() +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + add_compile_options("-Wno-ignored-attributes") +endif() + set(USE_PROF_API "1") if(NOT DEFINED LIBVDI_STATIC_DIR) From af90312867c06286e25186b089700ce41540c543 Mon Sep 17 00:00:00 2001 From: jiabaxie <57198431+jiabaxie@users.noreply.github.com> Date: Fri, 28 Feb 2020 06:16:12 -0500 Subject: [PATCH 057/154] Cleaned up error messages for HipEnvVarDriver test (#1825) There were several error messages that appeared even if the hipEnvVarDriver.exe test passes and executes successfully. Now it is cleaned up. The following are those instances: * When popen searches for directed_test directory but does not find it, it outputs an error, then finds the hipEnvVar at the same level. Currently the fix will prompt the test to only output an error if both searches for hipEnvVar fails. * When assertion is used towards the later half of the test, conditions were set to specifically hide the devices, resulting in No Hip Device detected in the latter half of the test. The fix will make these errors not appear as they are intended to not find any devices. Assertions themselves are untouched. HipEnvVarDriver.cpp has also been refactored. Reading HipEnvVar will now happen in a helper function for getDeviceNumber and getDevicePCIBusNumRemote, as the code to read HipEnvVar were really similar in them. --- tests/src/hipEnvVarDriver.cpp | 72 ++++++++++++++++++++--------------- tests/src/test_common.cpp | 2 + tests/src/test_common.h | 5 +++ 3 files changed, 48 insertions(+), 31 deletions(-) diff --git a/tests/src/hipEnvVarDriver.cpp b/tests/src/hipEnvVarDriver.cpp index 07379f0878..c970cb7674 100644 --- a/tests/src/hipEnvVarDriver.cpp +++ b/tests/src/hipEnvVarDriver.cpp @@ -34,42 +34,52 @@ OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWA using namespace std; -const string directed_dir = "directed_tests" + string(PATH_SEPERATOR_STR) + "hipEnvVar"; -const string dir = "." + string(PATH_SEPERATOR_STR) + "hipEnvVar"; +const string directed_dir = string(".") + PATH_SEPERATOR_STR + "directed_tests" + PATH_SEPERATOR_STR + "hipEnvVar"; +const string dir = string(".") + PATH_SEPERATOR_STR + "hipEnvVar"; -int getDeviceNumber() { - char buff[512]; - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - FILE* in = popen((directed_dir + " -c").c_str(), "r"); - if(fgets(buff, 512, in) == NULL){ - pclose(in); - //Check at same level - in = popen((dir + " -c").c_str(), "r"); +int readHipEnvVar(string flags, char* buff){ + + std::cout << "\nFinding hipEnvVar in " << directed_dir << "...\n"; + FILE* directed_in = popen((directed_dir + flags).c_str(), "r"); + + if(fgets(buff, 512, directed_in) == NULL){ + std::cout << "Finding hipEnvVar in " << dir << "...\n"; + FILE* in = popen((dir + flags).c_str(), "r"); if(fgets(buff, 512, in) == NULL){ + pclose(directed_in); pclose(in); return 1; } + pclose(in); + } + std::cout << "hipEnvVar Found!\n"; + pclose(directed_in); + return 0; +} + +int getDeviceNumber(bool print_err=true) { + char buff[512]; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + + if (readHipEnvVar(string(" -c"), buff)){ + strncpy(buff, "1", 512); + if (print_err){ + std::cerr << "The system cannot find hipEnvVar, using 1 as number of devices\n"; + } + } + if (print_err) { + std::cout << buff; } - cout << buff; - pclose(in); return atoi(buff); } // Query the current device ID remotely to hipEnvVar void getDevicePCIBusNumRemote(int deviceID, char* pciBusID) { std::this_thread::sleep_for(std::chrono::milliseconds(10)); - FILE* in = popen((directed_dir + " -d " + std::to_string(deviceID)).c_str(), "r"); - if(fgets(pciBusID, 100, in) == NULL){ - pclose(in); - //Check at same level - in = popen((dir + " -d").c_str(), "r"); - if(fgets(pciBusID, 100, in) == NULL){ - pclose(in); - return; - } + if (readHipEnvVar((" -d " + std::to_string(deviceID)), pciBusID)){ + std::cerr << "The system cannot find hipEnvVar\n"; } cout << pciBusID; - pclose(in); return; } @@ -78,15 +88,15 @@ void getDevicePCIBusNum(int deviceID, char* pciBusID) { hipDevice_t deviceT; hipDeviceGet(&deviceT, deviceID); - memset(pciBusID, 0, 100); - hipDeviceGetPCIBusId(pciBusID, 100, deviceT); + memset(pciBusID, 0, 512); + hipDeviceGetPCIBusId(pciBusID, 512, deviceT); } int main() { unsetenv(HIP_VISIBLE_DEVICES_STR); unsetenv(CUDA_VISIBLE_DEVICES_STR); std::vector devPCINum; - char pciBusID[100]; + char pciBusID[512]; // collect the device pci bus ID for all devices int totalDeviceNum = getDeviceNumber(); std::cout << "The total number of available devices is " << totalDeviceNum << std::endl @@ -116,27 +126,27 @@ int main() { // check when set an invalid device number setenv("HIP_VISIBLE_DEVICES", "1000,0,1", 1); setenv("CUDA_VISIBLE_DEVICES", "1000,0,1", 1); - assert(getDeviceNumber() == 0); + assert(getDeviceNumber(false) == 0); if (totalDeviceNum > 2) { setenv("HIP_VISIBLE_DEVICES", "0,1,1000,2", 1); setenv("CUDA_VISIBLE_DEVICES", "0,1,1000,2", 1); - assert(getDeviceNumber() == 2); + assert(getDeviceNumber(false) == 2); setenv("HIP_VISIBLE_DEVICES", "0,1,2", 1); setenv("CUDA_VISIBLE_DEVICES", "0,1,2", 1); - assert(getDeviceNumber() == 3); + assert(getDeviceNumber(false) == 3); // test if CUDA_VISIBLE_DEVICES will be accepted by the runtime unsetenv(HIP_VISIBLE_DEVICES_STR); unsetenv(CUDA_VISIBLE_DEVICES_STR); setenv("CUDA_VISIBLE_DEVICES", "0,1,2", 1); - assert(getDeviceNumber() == 3); + assert(getDeviceNumber(false) == 3); } setenv("HIP_VISIBLE_DEVICES", "-100,0,1", 1); setenv("CUDA_VISIBLE_DEVICES", "-100,0,1", 1); - assert(getDeviceNumber() == 0); + assert(getDeviceNumber(false) == 0); std::cout << "PASSED" << std::endl; return 0; -} +} \ No newline at end of file diff --git a/tests/src/test_common.cpp b/tests/src/test_common.cpp index e7a2622662..1c0dcc8c34 100644 --- a/tests/src/test_common.cpp +++ b/tests/src/test_common.cpp @@ -37,10 +37,12 @@ int p_tests = -1; /*which tests to run. Interpretation is left to each test. de const char* HIP_VISIBLE_DEVICES_STR = "HIP_VISIBLE_DEVICES="; const char* CUDA_VISIBLE_DEVICES_STR = "CUDA_VISIBLE_DEVICES="; const char* PATH_SEPERATOR_STR = "\\"; +const char* NULL_DEVICE = "NUL:"; #else const char* HIP_VISIBLE_DEVICES_STR = "HIP_VISIBLE_DEVICES"; const char* CUDA_VISIBLE_DEVICES_STR = "CUDA_VISIBLE_DEVICES"; const char* PATH_SEPERATOR_STR = "/"; +const char* NULL_DEVICE = "/dev/null"; #endif namespace HipTest { diff --git a/tests/src/test_common.h b/tests/src/test_common.h index 426ea846b1..7d8c39e74c 100644 --- a/tests/src/test_common.h +++ b/tests/src/test_common.h @@ -105,6 +105,10 @@ THE SOFTWARE. #define pclose(x) _pclose(x) #define setenv(x,y,z) _putenv_s(x,y) #define unsetenv _putenv +#define fileno(x) _fileno(x) +#define dup(x) _dup(x) +#define dup2(x,y) _dup2(x,y) +#define close(x) _close(x) #else #define aligned_free(x) free(x) #endif @@ -124,6 +128,7 @@ extern int p_tests; extern const char* HIP_VISIBLE_DEVICES_STR; extern const char* CUDA_VISIBLE_DEVICES_STR; extern const char* PATH_SEPERATOR_STR; +extern const char* NULL_DEVICE; // ********************* CPP section ********************* #ifdef __cplusplus From edc97f30731eef5f2bf67ef4e62988cfefeee826 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Fri, 28 Feb 2020 03:16:55 -0800 Subject: [PATCH 058/154] Add hipDrvOccupancyMaxActiveBlocksPerMultiprocessor[WithFlags] (#1854) Equivalent to cuOccupancyMaxActiveBlocksPerMultiprocessor[WithFlags]. --- ...A_Driver_API_functions_supported_by_HIP.md | 8 +++--- .../src/CUDA2HIP_Driver_API_functions.cpp | 4 +-- include/hip/hcc_detail/hip_runtime_api.h | 25 ++++++++++++++++++- src/hip_module.cpp | 18 +++++++++++++ 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md b/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md index 8956533ed0..e77997b0b6 100644 --- a/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md +++ b/docs/markdown/CUDA_Driver_API_functions_supported_by_HIP.md @@ -1126,10 +1126,10 @@ | **CUDA** | **HIP** |**CUDA version\***| |-----------------------------------------------------------|---------------------------------------------------------|------------------| -| `cuOccupancyMaxActiveBlocksPerMultiprocessor` |`hipOccupancyMaxActiveBlocksPerMultiprocessor` | -| `cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags` |`hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags` | -| `cuOccupancyMaxPotentialBlockSize` |`hipOccupancyMaxPotentialBlockSize` | -| `cuOccupancyMaxPotentialBlockSizeWithFlags` | | +| `cuOccupancyMaxActiveBlocksPerMultiprocessor` |`hipDrvOccupancyMaxActiveBlocksPerMultiprocessor` | +| `cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags` |`hipDrvOccupancyMaxActiveBlocksPerMultiprocessorWithFlags` | +| `cuOccupancyMaxPotentialBlockSize` |`hipOccupancyMaxPotentialBlockSize` | +| `cuOccupancyMaxPotentialBlockSizeWithFlags` | | ## **22. Texture Reference Management [DEPRECATED]** diff --git a/hipify-clang/src/CUDA2HIP_Driver_API_functions.cpp b/hipify-clang/src/CUDA2HIP_Driver_API_functions.cpp index 5d700631c4..ab07a10e93 100644 --- a/hipify-clang/src/CUDA2HIP_Driver_API_functions.cpp +++ b/hipify-clang/src/CUDA2HIP_Driver_API_functions.cpp @@ -545,9 +545,9 @@ const std::map CUDA_DRIVER_FUNCTION_MAP{ // 5.21. Occupancy // cudaOccupancyMaxActiveBlocksPerMultiprocessor - {"cuOccupancyMaxActiveBlocksPerMultiprocessor", {"hipOccupancyMaxActiveBlocksPerMultiprocessor", "", CONV_OCCUPANCY, API_DRIVER}}, + {"cuOccupancyMaxActiveBlocksPerMultiprocessor", {"hipDrvOccupancyMaxActiveBlocksPerMultiprocessor", "", CONV_OCCUPANCY, API_DRIVER}}, // cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags - {"cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", {"hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", "", CONV_OCCUPANCY, API_DRIVER}}, + {"cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", {"hipDrvOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", "", CONV_OCCUPANCY, API_DRIVER}}, // cudaOccupancyMaxPotentialBlockSize {"cuOccupancyMaxPotentialBlockSize", {"hipOccupancyMaxPotentialBlockSize", "", CONV_OCCUPANCY, API_DRIVER}}, // cudaOccupancyMaxPotentialBlockSizeWithFlags diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 76209ef6a7..2c6726c161 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -2959,7 +2959,18 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor( * @brief Returns occupancy for a device function. * * @param [out] numBlocks Returned occupancy - * @param [in] func Kernel function for which occupancy is calulated + * @param [in] func Kernel function (hipFunction) for which occupancy is calulated + * @param [in] blockSize Block size the kernel is intended to be launched with + * @param [in] dynSharedMemPerBlk dynamic shared memory usage (in bytes) intended for each block + */ +hipError_t hipDrvOccupancyMaxActiveBlocksPerMultiprocessor( + int* numBlocks, hipFunction_t f, int blockSize, size_t dynSharedMemPerBlk); + +/** + * @brief Returns occupancy for a device function. + * + * @param [out] numBlocks Returned occupancy + * @param [in] f Kernel function for which occupancy is calulated * @param [in] blockSize Block size the kernel is intended to be launched with * @param [in] dynSharedMemPerBlk dynamic shared memory usage (in bytes) intended for each block * @param [in] flags Extra flags for occupancy calculation (currently ignored) @@ -2967,6 +2978,18 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor( hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( uint32_t* numBlocks, hipFunction_t f, uint32_t blockSize, size_t dynSharedMemPerBlk, unsigned int flags __dparm(hipOccupancyDefault)); +/** + * @brief Returns occupancy for a device function. + * + * @param [out] numBlocks Returned occupancy + * @param [in] f Kernel function(hipFunction_t) for which occupancy is calulated + * @param [in] blockSize Block size the kernel is intended to be launched with + * @param [in] dynSharedMemPerBlk dynamic shared memory usage (in bytes) intended for each block + * @param [in] flags Extra flags for occupancy calculation (currently ignored) + */ +hipError_t hipDrvOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( + int* numBlocks, hipFunction_t f, int blockSize, size_t dynSharedMemPerBlk, unsigned int flags); + #if __HIP_VDI__ && !defined(__HCC__) /** * @brief Launches kernels on multiple devices and guarantees all specified kernels are dispatched diff --git a/src/hip_module.cpp b/src/hip_module.cpp index 44f0f108a6..a88abba9cb 100644 --- a/src/hip_module.cpp +++ b/src/hip_module.cpp @@ -1471,6 +1471,15 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor( tls, numBlocks, f, blockSize, dynSharedMemPerBlk)); } +hipError_t hipDrvOccupancyMaxActiveBlocksPerMultiprocessor( + int* numBlocks, hipFunction_t f, int blockSize, size_t dynSharedMemPerBlk) +{ + HIP_INIT_API(hipDrvOccupancyMaxActiveBlocksPerMultiprocessor, numBlocks, f, blockSize, dynSharedMemPerBlk); + + return ihipLogStatus(ihipOccupancyMaxActiveBlocksPerMultiprocessor( + tls, (uint32_t*) numBlocks, f, blockSize, dynSharedMemPerBlk)); +} + hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( uint32_t* numBlocks, hipFunction_t f, uint32_t blockSize, size_t dynSharedMemPerBlk, unsigned int flags) @@ -1481,6 +1490,15 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( tls, numBlocks, f, blockSize, dynSharedMemPerBlk)); } +hipError_t hipDrvOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( + int* numBlocks, hipFunction_t f, int blockSize, size_t dynSharedMemPerBlk, + unsigned int flags) +{ + HIP_INIT_API(hipDrvOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, numBlocks, f, blockSize, dynSharedMemPerBlk, flags); + return ihipLogStatus(ihipOccupancyMaxActiveBlocksPerMultiprocessor( + tls, (uint32_t*) numBlocks, f, blockSize, dynSharedMemPerBlk)); +} + hipError_t hipLaunchKernel( const void* func_addr, dim3 numBlocks, dim3 dimBlocks, void** args, size_t sharedMemBytes, hipStream_t stream) From 6c5fa32815650cc20a4f783d09b013610348a4d5 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Fri, 28 Feb 2020 03:17:15 -0800 Subject: [PATCH 059/154] Remove deprecated HIP markers (#1876) --- CMakeLists.txt | 20 -- bin/hipcc | 17 -- docs/markdown/hip_profiling.md | 279 ------------------ docs/markdown/hip_tracing.md | 72 +++++ include/hip/hip_profile.h | 18 -- packaging/hip-hcc.txt | 12 +- samples/2_Cookbook/2_Profiler/Makefile | 53 ---- .../2_Cookbook/2_Profiler/MatrixTranspose.cpp | 219 -------------- samples/2_Cookbook/2_Profiler/Readme.md | 47 --- src/hip_hcc.cpp | 44 +-- src/hip_hcc_internal.h | 41 +-- 11 files changed, 77 insertions(+), 745 deletions(-) delete mode 100644 docs/markdown/hip_profiling.md create mode 100644 docs/markdown/hip_tracing.md delete mode 100644 samples/2_Cookbook/2_Profiler/Makefile delete mode 100644 samples/2_Cookbook/2_Profiler/MatrixTranspose.cpp delete mode 100644 samples/2_Cookbook/2_Profiler/Readme.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a77735865..07d23130c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -197,19 +197,6 @@ if (NOT CPACK_SET_DESTDIR) set(CPACK_PACKAGING_INSTALL_PREFIX "/opt/rocm/hip" CACHE PATH "Default installation path of hcc installer package") endif (CPACK_SET_DESTDIR) -# Check if we need to enable ATP marker -if(NOT DEFINED COMPILE_HIP_ATP_MARKER) - if(NOT DEFINED ENV{COMPILE_HIP_ATP_MARKER}) - set(COMPILE_HIP_ATP_MARKER 0) - else() - set(COMPILE_HIP_ATP_MARKER $ENV{COMPILE_HIP_ATP_MARKER}) - message(WARNING "HIP Markers are deprecated, please use roctracer/rocTX marker APIs.") - endif() -else() - message(WARNING "HIP Markers are deprecated, please use roctracer/rocTX marker APIs.") -endif() -add_to_config(_buildInfo COMPILE_HIP_ATP_MARKER) - ############################# # Profiling API support ############################# @@ -267,10 +254,6 @@ endif () if(HIP_PLATFORM STREQUAL "hcc") include_directories(${PROJECT_SOURCE_DIR}/include) set(HIP_HCC_BUILD_FLAGS) - if(COMPILE_HIP_ATP_MARKER) - include_directories(/opt/rocm/profiler/CXLActivityLogger/include) - set(HIP_HCC_BUILD_FLAGS "${HIP_HCC_BUILD_FLAGS} -DCOMPILE_HIP_ATP_MARKER=1") - endif() # Add HIP_VERSION to CMAKE__FLAGS set(HIP_HCC_BUILD_FLAGS "${HIP_HCC_BUILD_FLAGS} -DHIP_VERSION_MAJOR=${HIP_VERSION_MAJOR} -DHIP_VERSION_MINOR=${HIP_VERSION_MINOR} -DHIP_VERSION_PATCH=${HIP_VERSION_GITDATE}") @@ -321,9 +304,6 @@ if(HIP_PLATFORM STREQUAL "hcc") set (CMAKE_BUILD_WITH_INSTALL_RPATH TRUE ) set (CMAKE_SKIP_BUILD_RPATH TRUE ) endif () - if(COMPILE_HIP_ATP_MARKER) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -L/opt/rocm/profiler/CXLActivityLogger/bin/x86_64 -lCXLActivityLogger") - endif() add_library(hip_hcc SHARED ${SOURCE_FILES_RUNTIME}) add_library(hip_hcc_static STATIC ${SOURCE_FILES_RUNTIME}) diff --git a/bin/hipcc b/bin/hipcc index 22923e5ec4..06a3f9e385 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -241,9 +241,6 @@ if ($HIP_PLATFORM eq "clang") { $HCC_VERSION_MAJOR=$HCC_VERSION; $HCC_VERSION_MAJOR=~s/\..*//; - $HIP_ATP_MARKER=$ENV{'HIP_ATP_MARKER'} // 1; - $marker_path = "$ROCM_PATH/profiler/CXLActivityLogger"; - # HCC* may be used to compile src/hip_hcc.o (and also feed the HIPCXXFLAGS below) $HCC = "$HCC_HOME/bin/hcc"; $HCCFLAGS = "-hc -D__HIPCC__ -isystem $HCC_HOME/include "; @@ -288,20 +285,6 @@ if ($HIP_PLATFORM eq "clang") { $HIPLDFLAGS .= " -L$HSA_PATH/lib -L$ROCM_PATH/lib -lhsa-runtime64 -lhc_am "; # $HIPLDFLAGS .= " -L$HCC_HOME/compiler/lib -lLLVMAMDGPUDesc -lLLVMAMDGPUUtils -lLLVMMC -lLLVMCore -lLLVMSupport "; - # Add trace marker library: - # TODO - once we cleanly separate the HIP API headers from HIP library headers this logic should move to CMakebuild option - apps do not need to see the marker library. - if ($HIP_ATP_MARKER) { - $marker_inc_path = "$marker_path/include"; - if (-e $marker_inc_path) { - $HIPCXXFLAGS .= " -isystem $marker_inc_path"; - } - } - - $marker_lib_path = "$marker_path/bin/x86_64"; - if (-e $marker_lib_path) { - $HIPLDFLAGS .= " -L$marker_lib_path -lCXLActivityLogger -Wl,--rpath=$marker_lib_path"; - } - if (not $isWindows) { $HIPLDFLAGS .= " -lm"; } diff --git a/docs/markdown/hip_profiling.md b/docs/markdown/hip_profiling.md deleted file mode 100644 index 28ed37e321..0000000000 --- a/docs/markdown/hip_profiling.md +++ /dev/null @@ -1,279 +0,0 @@ -# Profiling HIP Code - -This section describes the profiling and debugging capabilities that HIP provides. -Profiling information can viewed in the CodeXL visualization tool or printed directly to stderr as the application runs. -This document starts with some of the general capabilities of CodeXL and then describes some of the additional HIP marker and debug features. - - - -- [CodeXL Profiling](#codexl-profiling) - * [Collecting and Viewing Traces](#collecting-and-viewing-traces) - + [Using rocm-profiler timestamp profiling](#using-rocm-profiler-timestamp-profiling) - + [Using rocm-profiler performance counter collection:](#using-rocm-profiler-performance-counter-collection) - + [Using CodeXL to view profiling results:](#using-codexl-to-view-profiling-results) - + [More information on CodeXL](#more-information-on-codexl) - * [HIP Markers](#hip-markers) - + [Profiling HIP APIs](#profiling-hip-apis) - + [Adding markers to applications](#adding-markers-to-applications) - * [Additional HIP Profiling Features](#additional-hip-profiling-features) - + [Demangling C++ Kernel Names](#demangling-c-kernel-names) - + [Controlling when profiling starts and ends](#controlling-when-profiling-starts-and-ends) - + [Reducing timeline trace output file size](#reducing-timeline-trace-output-file-size) - + [How to enable profiling at HIP build time](#how-to-enable-profiling-at-hip-build-time) -- [Tracing and Debug](#tracing-and-debug) - * [Tracing HIP APIs](#tracing-hip-apis) - + [Color](#color) - - - -## CodeXL Profiling - -### Collecting and Viewing Traces - -#### Using rocm-profiler timestamp profiling -rocm-profiler is a command-line tool for tracing any application that uses ROCr API, including HCC and HIP. -rocm-profiler's timeline trace will show the beginning and end for all kernel commands, data transfer commands, and HSA Runtime (ROCr) API calls. The trace results are saved into a file, which by convention uses the "atp" extension. Here is an example that shows how to run the command-line profiler: -```shell -$ /opt/rocm/bin/rocm-profiler -o -A -T -``` - -#### Using rocm-profiler performance counter collection: -rocm-profiler can record performance counter information to provide greater insight inside a kernel, such as the memory bandwidth, ALU busy percentage, and cache statistics. -Collecting the common set of useful counters requires passing the counter configuration files for two passes: -``` -$ /opt/rocm/bin/rocm-profiler -C -O --counterfile /opt/rocm/profiler/counterfiles/counters_HSA_Fiji_pass1 --counterfile /opt/rocm/profiler/counterfiles/counters_HSA_Fiji_pass2 -``` - - -#### Using CodeXL to view profiling results: -The trace can be loaded and viewed in the CodeXL visualization tool: - -- Open the CodeXL GUI, create an new project, and switch to "Profile Mode": - - $ CodeXL & - - [File->New Project, leave fields as is, just click "OK"] - - [Profile->Switch to Profile Mode] -- Load timestamp tracing results into a timeline view: - - Right click on the project in the CodeXL Explorer view - - Click "Import Session..." - - Select to $HOME/apitrace.atp (or appropriate .atp file if you used another file name) - -- Load the performance counter results - - Right click on the project in the CodeXL Explorer view - - Click "Import Session..." - - Select $HOME/Session1.csv (or appropriate .csv file if you used another file name) - - -#### More information on CodeXL -rocm-profiler --help will show additional options and usage guidelines. - -See this [blog](http://gpuopen.com/getting-up-to-speed-with-the-codexl-gpu-profiler-and-radeon-open-compute/) for more information on profiling ROCm apps (including HIP) with CodeXL. - -The 2.2 version of Windows CodeXL does not correctly handle Linux line-endings. If you are collecting a trace on Linux and then viewing it with the 2.2 Windows CodeXL, first convert the line ending in the .atp file to Windows-style line endings. - -### HIP Markers -#### Profiling HIP APIs -HIP can generate markers at function beginning and end which are displayed on the CodeXL timeline view. -HIP 1.0 compiles marker support by default, and you can enable it by setting the HIP_PROFILE_API environment variable and then running the rocm-profiler: - -```shell - -# Use profile to generate timeline view: -export HIP_PROFILE_API=1 -$ /opt/rocm/bin/rocm-profiler -A -T - -Or -$ /opt/rocm/bin/rocm-profiler -e HIP_PROFILE_API=1 -A -T -``` - -HIP_PROFILE_API supports two levels of information. -- HIP_PROFILE_API=1 : Short format. Print name of API but no arguments. For example: -`hipMemcpy` -- HIP_PROFILE_API=2 : Long format. Print name of API + values of all function arguments. For example: -`hipMemcpy (0x7f32154db010, 0x50446e000, 4000000, hipMemcpyDeviceToHost)` - -#### Adding markers to applications - -Markers can be used to define application-specific events that will be recorded in the ATP file and displayed in the CodeXL GUI. -This can be particularly useful for visualizing how the higher-level phases of application behavior relate to the lower level HIP APIs, kernel launches, and data transfers. -For example, an instrumented machine learning framework could show the beginning and ending of each layer in the network. - -Markers have a specific begin and end time, and can be nested. Nested calls are displayed hierarchically in the CodeXL GUI, with each level of the hierarchy occupying a different row. - -The HIP APis are defined in "hip_profile.h": -``` -#include - -HIP_BEGIN_MARKER(const char *markerName, const char *groupName); -HIP_END_MARKER(); - -HIP_BEGIN_MARKER("Setup", "MyAppGroup"); -// ... -// application code for setup -// ... -HIP_END_MARKER(); -``` - -For C++ codes, HIP also provides a scoped marker which records the start time when constructed and the end time when the scoped marker is destructed at the end of the scope. This provides a convenient, single-line mechanism to record an event that neatly corresponds to a region of code. - -```cxx -void FunctionFoo(...) -{ - HIP_SCOPED_MARKER("FunctionFoo", "MyAppGroup"); // Marker starts recording here. - - // ... - // Function implementation - // ... - - // Marker destroyed here and records end time stamp. -}; -``` - -The HIP marker API is only supported on ROCm platform. The marker macros are defined on CUDA platforms and will compile, but are silently ignored at runtime. - -This [HIP sample](https://github.com/ROCm-Developer-Tools/HIP/tree/master/samples/2_Cookbook/2_Profiler) shows the profiler marker API used in a small application. - -More information on the marker API can be found in the profiler header file and PDF in a ROCm installation: -- /opt/rocm/profiler/CXLActivityLogger/include/CXLActivityLogger.h -- /opt/rocm/profiler/CXLActivityLogger/doc/CXLActivityLogger.pdf - -### Additional HIP Profiling Features -#### Demangling C++ Kernel Names -HIP includes the `hipdemangleatp` tool which can post-process an ATP file to "demangle" C++ names. -Mangled kernel names encode the C++ arguments and other information, and are guaranteed to be unique even for cases such as operator overloading. However, the mangled names can be quite verbose. For example: - -`ZZ39gemm_NoTransA_MICRO_NBK_M_N_K_TS16XMTS4RN2hc16accelerator_viewEPKflS3_lPfliiiiiiffEN3_EC__719__cxxamp_trampolineElililiiiiiiS3_iS3_S4_ff` - -`hipdemangleatp` will convert this into the more readable: -`gemm_NoTransA_MICRO_NBK_M_N_K_TS16XMTS4` - -The `hipdemangleatp` tool operates on the ATP file "in-place" and thus replaces the input file with the demangled version. - -``` -$ hipdemangleatp myfile.atp -``` - -The kernel name is also shown in some of the summary htlm files (Top10 kernels). These can be regenerated from the demangled ATP file by re-running rocm-profiler: -``` -$ rocm-profiler -T --atpfile myfile.atp -``` - -A future version of CodeXL may directly integrate demangle functionality. - - -#### Controlling when profiling starts and ends -hipProfilerStart() and hipProfilerEnd() can be inserted into an application to control which phases of the applications are profiled. -These APIs can be used to skip initialization code or to focus profiling on a desired region, and are particularly useful for large long-running applications. -See the API documentation for more information. These APIs work on both ROCm and CUDA paths. - -On ROCm, the following environment variables can be used to control when profiling occurs: - -``` -HIP_DB_START_API : Comma-separated list of tid.api_seq_num for when to start debug and profiling. -HIP_DB_STOP_API : Comma-separated list of tid.api_seq_num for when to stop debug and profiling. -``` - -HIP/ROCm assigns a monotonically increasing sequence number to the APIs called from each thread. The thread and API sequence number can be used in the above API to control when tracing starts and stops. These flags also control the HIP_DB messages (described below). - -When using these options, start the profiler with profiling disabled: -``` -# ROCm: -$ rocm-profiler --startdisabled ... - -# CUDA: -$ nvprof --profile-from-start-off ... -``` - -This feature is under development. - -#### Reducing timeline trace output file size -If the application is already recording the HIP APIs, the HSA APIs are somewhat redundant and the ATP file size can be substantially reduced by not recording these APIs. HIP includes a text file that lists all of the HSA APIs and can assist in this filtering: - -``` -$ rocm-profiler -F hip/bin/hsa-api-filter-cxl.txt -``` - -This file can be copied and edited to provide more selective HSA event recording. - - -#### How to enable profiling at HIP build time -Pre-built packages of HIP are not built with profiling support enabled.You must enable marker support manually when compiling HIP. - -1. Build HIP with ATP markers enabled -HIP pre-built packages are enabled with ATP marker support by default. -To enable ATP marker support when building HIP from source, use the option ```-DCOMPILE_HIP_ATP_MARKER=1``` during the cmake configure step. Build and install HIP. -```shell -$ mkdir build && cd build -$ cmake .. -DCOMPILE_HIP_ATP_MARKER -$ make install -``` - -2. Install ROCm-Profiler -Installing HIP from the [rocm](http://gpuopen.com/getting-started-with-boltzmann-components-platforms-installation/) pre-built packages, installs the ROCm-Profiler as well. -Alternatively, you can build ROCm-Profiler using the instructions [here](https://github.com/RadeonOpenCompute/ROCm-Profiler#building-the-rocm-profiler). - -3. Recompile the target application - -Then follow the steps above to collect a marker-enabled trace. - - -## Tracing and Debug - -### Tracing HIP APIs -The HIP runtime can print the HIP function strings to stderr using HIP_TRACE_API environment variable. -The trace prints two messages for each API - one at the beginning of the API call (line starts with "<<") and one at the end of the API call (line ends with ">>"). -Here's an example for one API followed by a description for the sections of the trace: - -``` -<> -``` - -- `<> -info: running on device gfx803 -info: allocate host mem ( 7.63 MB) -info: allocate device mem ( 7.63 MB) -<> -<> -info: copy Host2Device -<> -info: launch 'vector_square' kernel -1.5 hipLaunchKernel 'HIP_KERNEL_NAME(vector_square)' gridDim:{512,1,1} groupDim:{256,1,1} sharedMem:+0 stream#0.0 -info: copy Device2Host -<> -info: check result -PASSED! -``` - -HIP_TRACE_API supports multiple levels of debug information: - - 0x1 = print all HIP APIs. This is the most verbose setting; the flags below allow selecting a subset. - - 0x2 = print HIP APIs which initiate GPU kernel commands. Includes hipLaunchKernel, hipLaunchModuleKernel - - 0x4 = print HIP APIs which initiate GPU memory commands. Includes hipMemcpy*, hipMemset*. - - 0x8 = print HIP APIs which allocate or free memory. Includes hipMalloc, hipHostMalloc, hipFree, hipHostFree. - -These can be combined. For example, HIP_TRACE_API=6 shows a concise view of the HIP commands (both kernel and memory) that are sent to the GPU. - - -#### Color -Note this trace mode uses colors. "less -r" can handle raw control characters and will display the debug output in proper colors. -You can change the color used for the trace mode with the HIP_TRACE_API_COLOR environment variable. Possible values are None/Red/Green/Yellow/Blue/Magenta/Cyan/White. -None will disable use of color control codes for both the opening and closing and may be useful when saving the trace file or when a pure text trace is desired. - - - diff --git a/docs/markdown/hip_tracing.md b/docs/markdown/hip_tracing.md new file mode 100644 index 0000000000..40513f4e3c --- /dev/null +++ b/docs/markdown/hip_tracing.md @@ -0,0 +1,72 @@ +# Profiling HIP Code + +This section describes the tracing and debugging capabilities that HIP provides. + + +- [Tracing and Debug](#tracing-and-debug) + * [Tracing HIP APIs](#tracing-hip-apis) + + [Color](#color) + + + +## Tracing and Debug + +### Tracing HIP APIs +The HIP runtime can print the HIP function strings to stderr using HIP_TRACE_API environment variable. +The trace prints two messages for each API - one at the beginning of the API call (line starts with "<<") and one at the end of the API call (line ends with ">>"). +Here's an example for one API followed by a description for the sections of the trace: + +``` +<> +``` + +- `<> +info: running on device gfx803 +info: allocate host mem ( 7.63 MB) +info: allocate device mem ( 7.63 MB) +<> +<> +info: copy Host2Device +<> +info: launch 'vector_square' kernel +1.5 hipLaunchKernel 'HIP_KERNEL_NAME(vector_square)' gridDim:{512,1,1} groupDim:{256,1,1} sharedMem:+0 stream#0.0 +info: copy Device2Host +<> +info: check result +PASSED! +``` + +HIP_TRACE_API supports multiple levels of debug information: + - 0x1 = print all HIP APIs. This is the most verbose setting; the flags below allow selecting a subset. + - 0x2 = print HIP APIs which initiate GPU kernel commands. Includes hipLaunchKernel, hipLaunchModuleKernel + - 0x4 = print HIP APIs which initiate GPU memory commands. Includes hipMemcpy*, hipMemset*. + - 0x8 = print HIP APIs which allocate or free memory. Includes hipMalloc, hipHostMalloc, hipFree, hipHostFree. + +These can be combined. For example, HIP_TRACE_API=6 shows a concise view of the HIP commands (both kernel and memory) that are sent to the GPU. + + +#### Color +Note this trace mode uses colors. "less -r" can handle raw control characters and will display the debug output in proper colors. +You can change the color used for the trace mode with the HIP_TRACE_API_COLOR environment variable. Possible values are None/Red/Green/Yellow/Blue/Magenta/Cyan/White. +None will disable use of color control codes for both the opening and closing and may be useful when saving the trace file or when a pure text trace is desired. + + + diff --git a/include/hip/hip_profile.h b/include/hip/hip_profile.h index 95224af4a3..ff18239e44 100644 --- a/include/hip/hip_profile.h +++ b/include/hip/hip_profile.h @@ -1,16 +1,13 @@ /* Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -23,23 +20,8 @@ THE SOFTWARE. #ifndef HIP_INCLUDE_HIP_HIP_PROFILE_H #define HIP_INCLUDE_HIP_HIP_PROFILE_H -#warning "HIP Profiling through markers is deprecated, please check roctrace/rocTX support." - -#if not defined(ENABLE_HIP_PROFILE) -#define ENABLE_HIP_PROFILE 1 -#endif - -#if defined(__HIP_PLATFORM_HCC__) and (ENABLE_HIP_PROFILE == 1) -#warning "HIP Markers are deprecated and would be removed soon." -#include -#define HIP_SCOPED_MARKER(markerName, group) \ - amdtScopedMarker __scopedMarker(markerName, group, nullptr); -#define HIP_BEGIN_MARKER(markerName, group) amdtBeginMarker(markerName, group, nullptr); -#define HIP_END_MARKER() amdtEndMarker(); -#else #define HIP_SCOPED_MARKER(markerName, group) #define HIP_BEGIN_MARKER(markerName, group) #define HIP_END_MARKER() -#endif #endif diff --git a/packaging/hip-hcc.txt b/packaging/hip-hcc.txt index 63f3d73e67..21e138e1ed 100644 --- a/packaging/hip-hcc.txt +++ b/packaging/hip-hcc.txt @@ -36,11 +36,7 @@ set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR} set(CPACK_GENERATOR "TGZ;DEB;RPM") set(CPACK_BINARY_DEB "ON") set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${PROJECT_BINARY_DIR}/postinst;${PROJECT_BINARY_DIR}/prerm") -if(@COMPILE_HIP_ATP_MARKER@) - set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip-base (= ${CPACK_PACKAGE_VERSION}), ${HCC_PACKAGE_NAME} (= @HCC_PACKAGE_VERSION@), rocm-profiler, comgr (>= 1.1)") -else() - set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip-base (= ${CPACK_PACKAGE_VERSION}), ${HCC_PACKAGE_NAME} (= @HCC_PACKAGE_VERSION@), comgr (>= 1.1)") -endif() +set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip-base (= ${CPACK_PACKAGE_VERSION}), ${HCC_PACKAGE_NAME} (= @HCC_PACKAGE_VERSION@), comgr (>= 1.1)") set(CPACK_DEBIAN_PACKAGE_PROVIDES "hip_hcc") set(CPACK_DEBIAN_PACKAGE_REPLACES "hip_hcc") set(CPACK_DEBIAN_PACKAGE_CONFLICTS "hip_hcc") @@ -50,11 +46,7 @@ set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${PROJECT_BINARY_DIR}/postinst") set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${PROJECT_BINARY_DIR}/prerm") set(CPACK_RPM_PACKAGE_AUTOREQPROV " no") string(REPLACE "-" "_" HIP_BASE_VERSION ${CPACK_PACKAGE_VERSION}) -if(@COMPILE_HIP_ATP_MARKER@) - set(CPACK_RPM_PACKAGE_REQUIRES "hip-base = ${HIP_BASE_VERSION}, ${HCC_PACKAGE_NAME} = @HCC_PACKAGE_VERSION@, rocm-profiler, comgr >= 1.1") -else() - set(CPACK_RPM_PACKAGE_REQUIRES "hip-base = ${HIP_BASE_VERSION}, ${HCC_PACKAGE_NAME} = @HCC_PACKAGE_VERSION@, comgr >= 1.1") -endif() +set(CPACK_RPM_PACKAGE_REQUIRES "hip-base = ${HIP_BASE_VERSION}, ${HCC_PACKAGE_NAME} = @HCC_PACKAGE_VERSION@, comgr >= 1.1") set(CPACK_RPM_PACKAGE_OBSOLETES "hip_hcc") set(CPACK_RPM_PACKAGE_CONFLICTS "hip_hcc") set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/opt") diff --git a/samples/2_Cookbook/2_Profiler/Makefile b/samples/2_Cookbook/2_Profiler/Makefile deleted file mode 100644 index db2d008182..0000000000 --- a/samples/2_Cookbook/2_Profiler/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -HIP_PATH?= $(wildcard /opt/rocm/hip) - -HIPCC=$(HIP_PATH)/bin/hipcc - - -HIPPROFILER=/opt/rocm/bin/rocm-profiler -PROFILER_OPT=-A -o MT.atp -e HIP_PROFILE_API=1 -HIPPROFILER_POST_CMD=$(HIP_PATH)/bin/hipdemangleatp MT.atp - -TARGET=hcc - -SOURCES = MatrixTranspose.cpp -OBJECTS = $(SOURCES:.cpp=.o) - -EXECUTABLE=./MatrixTranspose - -.PHONY: test - - -all: $(EXECUTABLE) profile - - - -OPT =-g -CXXFLAGS =$(OPT) -CXX=$(HIPCC) - - -$(EXECUTABLE): $(OBJECTS) - $(HIPCC) $(OBJECTS) -o $@ - - -profile: $(EXECUTABLE) - $(HIPPROFILER) $(PROFILER_OPT) $(EXECUTABLE) - $(HIPPROFILER_POST_CMD) - - -# Pass option to control start and stop iterations for profiling - see MatrixTranspose.cpp for implementation: -# Note we start profiler in --startdisabled mode - no timing collected until app enabled it via hipProfilerStart() -profile_trigger: $(EXECUTABLE) - $(HIPPROFILER) $(PROFILER_OPT) --startdisabled $(EXECUTABLE) 3 6 - $(HIPPROFILER_POST_CMD) - - -run: $(EXECUTABLE) - $(EXECUTABLE) - - -clean: - rm -f $(EXECUTABLE) - rm -f $(OBJECTS) - rm -f $(HIP_PATH)/src/*.o - diff --git a/samples/2_Cookbook/2_Profiler/MatrixTranspose.cpp b/samples/2_Cookbook/2_Profiler/MatrixTranspose.cpp deleted file mode 100644 index 69266e1288..0000000000 --- a/samples/2_Cookbook/2_Profiler/MatrixTranspose.cpp +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright (c) 2015-present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#include - -// hip header file -#include "hip/hip_runtime.h" -#include "hip/hip_profile.h" - -#define WIDTH 1024 - -#define NUM (WIDTH * WIDTH) - -#define THREADS_PER_BLOCK_X 4 -#define THREADS_PER_BLOCK_Y 4 -#define THREADS_PER_BLOCK_Z 1 - -#define ITERATIONS 10 - -// Cmdline parms to control start and stop triggers -int startTriggerIteration = -1; -int stopTriggerIteration = -1; - -// Device (Kernel) function, it must be void -__global__ void matrixTranspose(float* out, float* in, const int width) { - int x = hipBlockDim_x * hipBlockIdx_x + hipThreadIdx_x; - int y = hipBlockDim_y * hipBlockIdx_y + hipThreadIdx_y; - - out[y * width + x] = in[x * width + y]; -} - -// CPU implementation of matrix transpose -void matrixTransposeCPUReference(float* output, float* input, const unsigned int width) { - for (unsigned int j = 0; j < width; j++) { - for (unsigned int i = 0; i < width; i++) { - output[i * width + j] = input[j * width + i]; - } - } -} - - -// Use a separate function to demonstrate how to use function name as part of scoped marker: -void runGPU(float* Matrix, float* TransposeMatrix, float* gpuMatrix, float* gpuTransposeMatrix) { - // __func__ is a standard C++ macro which expands to the name of the function, in this case - // "runGPU" - HIP_SCOPED_MARKER(__func__, "MyGroup"); - - for (int i = 0; i < ITERATIONS; i++) { - if (i == startTriggerIteration) { - hipProfilerStart(); - } - if (i == stopTriggerIteration) { - hipProfilerStop(); - } - - float eventMs = 0.0f; - - hipEvent_t start, stop; - hipEventCreate(&start); - hipEventCreate(&stop); - - - // Record the start event - hipEventRecord(start, NULL); - - // Memory transfer from host to device - hipMemcpy(gpuMatrix, Matrix, NUM * sizeof(float), hipMemcpyHostToDevice); - - // Record the stop event - hipEventRecord(stop, NULL); - hipEventSynchronize(stop); - - hipEventElapsedTime(&eventMs, start, stop); - - printf("hipMemcpyHostToDevice time taken = %6.3fms\n", eventMs); - - // Record the start event - hipEventRecord(start, NULL); - - // Lauching kernel from host - hipLaunchKernelGGL(matrixTranspose, - dim3(WIDTH / THREADS_PER_BLOCK_X, WIDTH / THREADS_PER_BLOCK_Y), - dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y), 0, 0, gpuTransposeMatrix, - gpuMatrix, WIDTH); - - // Record the stop event - hipEventRecord(stop, NULL); - hipEventSynchronize(stop); - hipEventElapsedTime(&eventMs, start, stop); - - printf("kernel Execution time = %6.3fms\n", eventMs); - - // Record the start event - hipEventRecord(start, NULL); - - // Memory transfer from device to host - hipMemcpy(TransposeMatrix, gpuTransposeMatrix, NUM * sizeof(float), hipMemcpyDeviceToHost); - - // Record the stop event - hipEventRecord(stop, NULL); - hipEventSynchronize(stop); - - hipEventElapsedTime(&eventMs, start, stop); - - printf("hipMemcpyDeviceToHost time taken = %6.3fms\n", eventMs); - } -}; - - -int main(int argc, char* argv[]) { - if (argc >= 2) { - startTriggerIteration = atoi(argv[1]); - printf("info : will start tracing at iteration:%d\n", startTriggerIteration); - } - if (argc >= 3) { - stopTriggerIteration = atoi(argv[2]); - printf("info : will stop tracing at iteration:%d\n", stopTriggerIteration); - } - - float* Matrix; - float* TransposeMatrix; - float* cpuTransposeMatrix; - - float* gpuMatrix; - float* gpuTransposeMatrix; - - hipDeviceProp_t devProp; - hipGetDeviceProperties(&devProp, 0); - - std::cout << "Device name " << devProp.name << std::endl; - - { - // Show example of how to create a "scoped marker". - // The scoped marker records the time spent inside the { scope } of the marker - the begin - // timestamp is at the beginning of the code scope, and the end is recorded when the SCOPE - // exits. This can be viewed in CodeXL timeline relative to other GPU and CPU events. This - // marker captures the time spent in setup including host allocation, initialization, and - // device memory allocation. - HIP_SCOPED_MARKER("Setup", "MyGroup"); - - - Matrix = (float*)malloc(NUM * sizeof(float)); - TransposeMatrix = (float*)malloc(NUM * sizeof(float)); - cpuTransposeMatrix = (float*)malloc(NUM * sizeof(float)); - - // initialize the input data - for (int i = 0; i < NUM; i++) { - Matrix[i] = (float)i * 10.0f; - } - - - // allocate the memory on the device side - hipMalloc((void**)&gpuMatrix, NUM * sizeof(float)); - hipMalloc((void**)&gpuTransposeMatrix, NUM * sizeof(float)); - - // FYI, the scoped-marker will be destroyed here when the scope exits, and will record its - // "end" timestamp. - } - - runGPU(Matrix, TransposeMatrix, gpuMatrix, gpuTransposeMatrix); - - - // show how to use explicit begin/end markers: - // We begin the timed region with HIP_BEGIN_MARKER, passing in the markerName and group: - // The region will stop when HIP_END_MARKER is called - // This is another way to mark begin/end - as an alternative to scoped markers. - HIP_BEGIN_MARKER("Check&TearDown", "MyGroup"); - - int errors = 0; - - // CPU MatrixTranspose computation - matrixTransposeCPUReference(cpuTransposeMatrix, Matrix, WIDTH); - - // verify the results - double eps = 1.0E-6; - for (int i = 0; i < NUM; i++) { - if (std::abs(TransposeMatrix[i] - cpuTransposeMatrix[i]) > eps) { - errors++; - } - } - if (errors != 0) { - printf("FAILED: %d errors\n", errors); - } else { - printf("PASSED!\n"); - } - - // free the resources on device side - hipFree(gpuMatrix); - hipFree(gpuTransposeMatrix); - - // free the resources on host side - free(Matrix); - free(TransposeMatrix); - free(cpuTransposeMatrix); - - // This ends the last marker started in this thread, in this case "Check&TearDown" - HIP_END_MARKER(); - - return errors; -} diff --git a/samples/2_Cookbook/2_Profiler/Readme.md b/samples/2_Cookbook/2_Profiler/Readme.md deleted file mode 100644 index 8b32beb348..0000000000 --- a/samples/2_Cookbook/2_Profiler/Readme.md +++ /dev/null @@ -1,47 +0,0 @@ -## Using hipEvents to measure performance ### - -This tutorial is follow-up of the previous two tutorial where we learn how to write our first hip program, in which we compute Matrix Transpose and in second one, we added feature to measure time taken for memory transfer and kernel execution. In this tutorial, we'll explain how to use the codexl/rocm-profiler for hip timeline tracing. Also, we will augment the source code with additional markers so we can see the high-level application flow alongside the information that CodeXL automatically collects. - - -## Introduction: - -CodeXL and rocm-profiler are the tool used for profiling the application, which is of prominent use in optimizing the application by means of finding the memory bottlenecks and etc. - -## Requirement: -[CodeXL Installation](http://gpuopen.com/compute-product/codexl/) - -## prerequiste knowledge: - -Programmers familiar with CUDA, OpenCL will be able to quickly learn and start coding with the HIP API. In case you are not, don't worry. You choose to start with the best one. We'll be explaining everything assuming you are completely new to gpgpu programming. - -## Simple Matrix Transpose - -We will be using the Simple Matrix Transpose source code from the previous tutorial as it is. - -## Using CodeXL markers for HIP Functions - -HIP can generate markers at function being/end which are displayed on the CodeXL timeline view. To do this, you need to install ROCm-Profiler and enable HIP to generate the markers: - -1. Install ROCm-Profiler Installing HIP from the rocm pre-built packages, installs the ROCm-Profiler as well. Alternatively, you can build ROCm-Profiler using the instructions given below. - - -2. Run with profiler enabled to generate ATP file. -(These steps are also captured in the Makefile) -The HIP_PROFILE_API enables display of the HIP APIs on the CodeXL trimeline view. -`/opt/rocm/bin/rocm-profiler -o -A -e HIP_PROFILE_API=1 ` - -##Using HIP_TRACE_API - -You can also print the HIP function strings to stderr using HIP_TRACE_API environment variable. This can also be combined with the more detailed debug information provided by the HIP_DB switch. For example: -`HIP_TRACE_API=1 HIP_DB=0x2 ./myHipApp` -Note this trace mode uses colors. "less -r" can handle raw control characters and will display the debug output in proper colors. - -## More Info: -- [HIP FAQ](https://github.com/ROCm-Developer-Tools/HIP/blob/master/docs/markdown/hip_faq.md) -- [HIP Kernel Language](https://github.com/ROCm-Developer-Tools/HIP/blob/master/docs/markdown/hip_kernel_language.md) -- [HIP Runtime API (Doxygen)](http://rocm-developer-tools.github.io/HIP) -- [HIP Porting Guide](https://github.com/ROCm-Developer-Tools/HIP/blob/master/docs/markdown/hip_porting_guide.md) -- [HIP Terminology](https://github.com/ROCm-Developer-Tools/HIP/blob/master/docs/markdown/hip_terms.md) (including Rosetta Stone of GPU computing terms across CUDA/HIP/HC/AMP/OpenL) -- [HIPIFY](https://github.com/ROCm-Developer-Tools/HIP/blob/master/hipify-clang/README.md) -- [Developer/CONTRIBUTING Info](https://github.com/ROCm-Developer-Tools/HIP/blob/master/CONTRIBUTING.md) -- [Release Notes](https://github.com/ROCm-Developer-Tools/HIP/blob/master/RELEASE.md) diff --git a/src/hip_hcc.cpp b/src/hip_hcc.cpp index af40b29ea9..c9688408c8 100644 --- a/src/hip_hcc.cpp +++ b/src/hip_hcc.cpp @@ -72,7 +72,6 @@ int HIP_API_BLOCKING = 0; int HIP_PRINT_ENV = 0; int HIP_TRACE_API = 0; std::string HIP_TRACE_API_COLOR("green"); -int HIP_PROFILE_API = 0; // TODO - DB_START/STOP need more testing. std::string HIP_DB_START_API; @@ -150,12 +149,10 @@ uint64_t recordApiTrace(TlsData *tls, std::string* fullStr, const std::string& a if ((tid < g_dbStartTriggers.size()) && (apiSeqNum >= g_dbStartTriggers[tid].nextTrigger())) { printf("info: resume profiling at %lu\n", apiSeqNum); - RESUME_PROFILING; g_dbStartTriggers.pop_back(); }; if ((tid < g_dbStopTriggers.size()) && (apiSeqNum >= g_dbStopTriggers[tid].nextTrigger())) { printf("info: stop profiling at %lu\n", apiSeqNum); - STOP_PROFILING; g_dbStopTriggers.pop_back(); }; @@ -1295,9 +1292,6 @@ void HipReadEnv() { "executes."); READ_ENV_S(release, HIP_TRACE_API_COLOR, 0, "Color to use for HIP_API. None/Red/Green/Yellow/Blue/Magenta/Cyan/White"); - READ_ENV_I(release, HIP_PROFILE_API, 0, - "Add HIP API markers to ATP file generated with CodeXL. 0x1=short API name, " - "0x2=full API name including args."); READ_ENV_S(release, HIP_DB_START_API, 0, "Comma-separated list of tid.api_seq_num for when to start debug and profiling."); READ_ENV_S(release, HIP_DB_STOP_API, 0, @@ -1373,14 +1367,6 @@ void HipReadEnv() { HIP_DB |= 0x1; } - if (HIP_PROFILE_API && !COMPILE_HIP_ATP_MARKER) { - fprintf(stderr, - "warning: env var HIP_PROFILE_API=0x%x but COMPILE_HIP_ATP_MARKER=0. (perhaps " - "enable COMPILE_HIP_ATP_MARKER in src code before compiling?)\n", - HIP_PROFILE_API); - HIP_PROFILE_API = 0; - } - if (HIP_DB) { fprintf(stderr, "HIP_DB=0x%x [%s]\n", HIP_DB, HIP_DB_string(HIP_DB).c_str()); } @@ -1424,11 +1410,6 @@ void HipReadEnv() { // This function creates a vector with only the GPU accelerators. // It is called with C++11 call_once, which provided thread-safety. void ihipInit() { -#if COMPILE_HIP_ATP_MARKER - amdtInitializeActivityLogger(); - amdtScopedMarker("ihipInit", "HIP", NULL); -#endif - HipReadEnv(); @@ -1618,7 +1599,7 @@ hipStream_t ihipSyncAndResolveStream(hipStream_t stream, bool lockAcquired) { void ihipPrintKernelLaunch(const char* kernelName, const grid_launch_parm* lp, const hipStream_t stream) { - if ((HIP_TRACE_API & (1 << TRACE_KCMD)) || HIP_PROFILE_API || + if ((HIP_TRACE_API & (1 << TRACE_KCMD)) || (COMPILE_HIP_DB & HIP_TRACE_API)) { GET_TLS(); std::stringstream os; @@ -1631,14 +1612,6 @@ void ihipPrintKernelLaunch(const char* kernelName, const grid_launch_parm* lp, std::string fullStr; recordApiTrace(tls, &fullStr, os.str()); } - - if (HIP_PROFILE_API == 0x1) { - std::string shortAtpString("hipLaunchKernel:"); - shortAtpString += kernelName; - MARKER_BEGIN(shortAtpString.c_str(), "HIP"); - } else if (HIP_PROFILE_API == 0x2) { - MARKER_BEGIN(os.str().c_str(), "HIP"); - } } } @@ -1697,9 +1670,6 @@ void ihipPostLaunchKernel(const char* kernelName, hipStream_t stream, grid_launc tprintf(DB_SYNC, "ihipPostLaunchKernel, unlocking stream\n"); stream->lockclose_postKernelCommand(kernelName, lp.av, unlockPostponed); - if (HIP_PROFILE_API) { - MARKER_END(); - } } //================================================================================================= @@ -2481,29 +2451,17 @@ bool ihipStream_t::locked_copy2DAsync(void* dst, const void* src, size_t width, return retStatus; } -//------------------------------------------------------------------------------------------------- -//------------------------------------------------------------------------------------------------- -// Profiler, really these should live elsewhere: hipError_t hipProfilerStart() { HIP_INIT_API(hipProfilerStart); -#if COMPILE_HIP_ATP_MARKER - amdtResumeProfiling(AMDT_ALL_PROFILING); -#endif - return ihipLogStatus(hipSuccess); }; hipError_t hipProfilerStop() { HIP_INIT_API(hipProfilerStop); -#if COMPILE_HIP_ATP_MARKER - amdtStopProfiling(AMDT_ALL_PROFILING); -#endif - return ihipLogStatus(hipSuccess); }; - //------------------------------------------------------------------------------------------------- //------------------------------------------------------------------------------------------------- // HCC-specific accessor functions: diff --git a/src/hip_hcc_internal.h b/src/hip_hcc_internal.h index 0510015c42..ac63f49dba 100644 --- a/src/hip_hcc_internal.h +++ b/src/hip_hcc_internal.h @@ -63,7 +63,6 @@ extern int HIP_LAUNCH_BLOCKING; extern int HIP_API_BLOCKING; extern int HIP_PRINT_ENV; -extern int HIP_PROFILE_API; // extern int HIP_TRACE_API; extern int HIP_ATP; extern int HIP_DB; @@ -250,34 +249,6 @@ extern const char* API_COLOR_END; // Must be enabled at runtime with HIP_TRACE_API #define COMPILE_HIP_TRACE_API 0x3 - -// Compile code that generates trace markers for CodeXL ATP at HIP function begin/end. -// ATP is standard CodeXL format that includes timestamps for kernels, HSA RT APIs, and HIP APIs. -#ifndef COMPILE_HIP_ATP_MARKER -#define COMPILE_HIP_ATP_MARKER 0 -#endif - - -// Compile support for trace markers that are displayed on CodeXL GUI at start/stop of each function -// boundary. -// TODO - currently we print the trace message at the beginning. if we waited, we could also -// tls->tidInfo return codes, and any values returned through ptr-to-args (ie the pointers allocated -// by hipMalloc). -#if COMPILE_HIP_ATP_MARKER -#include "CXLActivityLogger.h" -#define MARKER_BEGIN(markerName, group) amdtBeginMarker(markerName, group, nullptr); -#define MARKER_END() amdtEndMarker(); -#define RESUME_PROFILING amdtResumeProfiling(AMDT_ALL_PROFILING); -#define STOP_PROFILING amdtStopProfiling(AMDT_ALL_PROFILING); -#else -// Swallow scoped markers: -#define MARKER_BEGIN(markerName, group) -#define MARKER_END() -#define RESUME_PROFILING -#define STOP_PROFILING -#endif - - //--- // HIP Trace modes - use with HIP_TRACE_API=... #define TRACE_ALL 0 // 0x01 @@ -336,22 +307,17 @@ static inline uint64_t getTicks() { return hc::get_system_ticks(); } //--- extern uint64_t recordApiTrace(TlsData *tls, std::string* fullStr, const std::string& apiStr); -#if COMPILE_HIP_ATP_MARKER || (COMPILE_HIP_TRACE_API & 0x1) +#if (COMPILE_HIP_TRACE_API & 0x1) #define API_TRACE(forceTrace, ...) \ GET_TLS(); \ uint64_t hipApiStartTick = 0; \ { \ tls->tidInfo.incApiSeqNum(); \ if (forceTrace || \ - (HIP_PROFILE_API || (COMPILE_HIP_DB && (HIP_TRACE_API & (1 << TRACE_ALL))))) { \ + (COMPILE_HIP_DB && (HIP_TRACE_API & (1 << TRACE_ALL)))) { \ std::string apiStr = std::string(__func__) + " (" + ToString(__VA_ARGS__) + ')'; \ std::string fullStr; \ hipApiStartTick = recordApiTrace(tls, &fullStr, apiStr); \ - if (HIP_PROFILE_API == 0x1) { \ - MARKER_BEGIN(__func__, "HIP") \ - } else if (HIP_PROFILE_API == 0x2) { \ - MARKER_BEGIN(fullStr.c_str(), "HIP"); \ - } \ } \ } @@ -398,9 +364,6 @@ extern uint64_t recordApiTrace(TlsData *tls, std::string* fullStr, const std::st tls->tidInfo.apiSeqNum(), __func__, localHipStatus, \ ihipErrorString(localHipStatus), ticks, API_COLOR_END); \ } \ - if (HIP_PROFILE_API) { \ - MARKER_END(); \ - } \ localHipStatus; \ }) From 3e1f41c165aee1cee534687c98c0e315d6c41051 Mon Sep 17 00:00:00 2001 From: saleelk Date: Fri, 28 Feb 2020 03:17:29 -0800 Subject: [PATCH 060/154] Fix HIPRTC headers to export C style symbols (#1879) --- include/hip/hcc_detail/hiprtc.h | 66 ++++------- src/hiprtc.cpp | 64 ++++++----- tests/src/hiprtc/hiprtcGetTypeName.cpp | 147 ------------------------- 3 files changed, 58 insertions(+), 219 deletions(-) delete mode 100644 tests/src/hiprtc/hiprtcGetTypeName.cpp diff --git a/include/hip/hcc_detail/hiprtc.h b/include/hip/hcc_detail/hiprtc.h index 26d3129dbc..624f1ea157 100644 --- a/include/hip/hcc_detail/hiprtc.h +++ b/include/hip/hcc_detail/hiprtc.h @@ -19,10 +19,14 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#pragma once +#ifndef HIPRTC_H +#define HIPRTC_H -#include -#include +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include enum hiprtcResult { HIPRTC_SUCCESS = 0, @@ -41,29 +45,22 @@ enum hiprtcResult { const char* hiprtcGetErrorString(hiprtcResult result); -inline -hiprtcResult hiprtcVersion(int* major, int* minor) noexcept -{ // TODO: NVRTC versioning is somewhat unclear. - if (!major || !minor) return HIPRTC_ERROR_INVALID_INPUT; - // TODO: this should be generic / set by the build infrastructure. - *major = 9; - *minor = 0; +hiprtcResult hiprtcVersion(int* major, int* minor); - return HIPRTC_SUCCESS; -} - -struct _hiprtcProgram; -using hiprtcProgram = _hiprtcProgram*; +typedef struct _hiprtcProgram* hiprtcProgram; hiprtcResult hiprtcAddNameExpression(hiprtcProgram prog, const char* name_expression); -hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, int numOptions, +hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, + int numOptions, const char** options); -hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, const char* src, - const char* name, int numHeaders, +hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, + const char* src, + const char* name, + int numHeaders, const char** headers, const char** includeNames); @@ -76,37 +73,14 @@ hiprtcResult hiprtcGetLoweredName(hiprtcProgram prog, hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* log); hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog, - std::size_t* logSizeRet); + size_t* logSizeRet); hiprtcResult hiprtcGetCode(hiprtcProgram prog, char* code); -hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, std::size_t* codeSizeRet); +hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, size_t* codeSizeRet); -namespace hip_impl -{ - char* demangle(const char* mangled_expression); +#ifdef __cplusplus } +#endif /* __cplusplus */ -#if defined(HIPRTC_GET_TYPE_NAME) - #include - - #if defined(_WIN32) - #include - - template - hiprtcResult hiprtcGetTypeName(std::string*) = delete; - #else - template - inline - hiprtcResult hiprtcGetTypeName(std::string* result) - { - if (!result) return HIPRTC_ERROR_INVALID_INPUT; - - char * res= hip_impl::demangle(typeid(T).name()); - result->assign(res == nullptr ? "" : res); - std::free(res); - return (result->empty()) ? HIPRTC_ERROR_INTERNAL_ERROR : - HIPRTC_SUCCESS; - } - #endif -#endif +#endif //HIPRTC_H diff --git a/src/hiprtc.cpp b/src/hiprtc.cpp index 5198bf0cbb..3c7fe6e78c 100644 --- a/src/hiprtc.cpp +++ b/src/hiprtc.cpp @@ -50,7 +50,7 @@ THE SOFTWARE. #include #include -const char* hiprtcGetErrorString(hiprtcResult x) +extern "C" const char* hiprtcGetErrorString(hiprtcResult x) { switch (x) { case HIPRTC_SUCCESS: @@ -95,6 +95,21 @@ inline bool fileExists (const std::string& name) { } } // namespace hip_impl +namespace +{ + char* demangle(const char* x) + { + if (!x) return nullptr; + + int s{}; + char* tmp = abi::__cxa_demangle(x, nullptr, nullptr, &s); + + if (s != 0) return nullptr; + + return tmp; + } +} // Unnamed namespace. + namespace { struct Symbol { @@ -158,7 +173,7 @@ struct _hiprtcProgram { { using namespace std; - char* demangled = hip_impl::demangle(name.c_str()); + char* demangled = demangle(name.c_str()); name.assign(demangled == nullptr ? "" : demangled); free(demangled); @@ -352,7 +367,7 @@ namespace } } // Unnamed namespace. -hiprtcResult hiprtcAddNameExpression(hiprtcProgram p, const char* n) +extern "C" hiprtcResult hiprtcAddNameExpression(hiprtcProgram p, const char* n) { if (!n) return HIPRTC_ERROR_INVALID_INPUT; if (!isValidProgram(p)) return HIPRTC_ERROR_INVALID_PROGRAM; @@ -413,21 +428,6 @@ namespace }; } // Unnamed namespace. -namespace hip_impl -{ - char* demangle(const char* x) - { - if (!x) return nullptr; - - int s{}; - char* tmp = abi::__cxa_demangle(x, nullptr, nullptr, &s); - - if (s != 0) return nullptr; - - return tmp; - } -} // Namespace hip_impl. - namespace { const std::string& defaultTarget() @@ -492,7 +492,7 @@ namespace } } // Unnamed namespace. -hiprtcResult hiprtcCompileProgram(hiprtcProgram p, int n, const char** o) +extern "C" hiprtcResult hiprtcCompileProgram(hiprtcProgram p, int n, const char** o) { using namespace std; @@ -530,7 +530,7 @@ hiprtcResult hiprtcCompileProgram(hiprtcProgram p, int n, const char** o) return HIPRTC_SUCCESS; } -hiprtcResult hiprtcCreateProgram(hiprtcProgram* p, const char* src, +extern "C" hiprtcResult hiprtcCreateProgram(hiprtcProgram* p, const char* src, const char* name, int n, const char** hdrs, const char** incs) { @@ -548,14 +548,14 @@ hiprtcResult hiprtcCreateProgram(hiprtcProgram* p, const char* src, return HIPRTC_SUCCESS; } -hiprtcResult hiprtcDestroyProgram(hiprtcProgram* p) +extern "C" hiprtcResult hiprtcDestroyProgram(hiprtcProgram* p) { if (!p) return HIPRTC_SUCCESS; return _hiprtcProgram::destroy(*p); } -hiprtcResult hiprtcGetLoweredName(hiprtcProgram p, const char* n, +extern "C" hiprtcResult hiprtcGetLoweredName(hiprtcProgram p, const char* n, const char** ln) { using namespace std; @@ -576,7 +576,7 @@ hiprtcResult hiprtcGetLoweredName(hiprtcProgram p, const char* n, return HIPRTC_SUCCESS; } -hiprtcResult hiprtcGetProgramLog(hiprtcProgram p, char* l) +extern "C" hiprtcResult hiprtcGetProgramLog(hiprtcProgram p, char* l) { if (!l) return HIPRTC_ERROR_INVALID_INPUT; if (!isValidProgram(p)) return HIPRTC_ERROR_INVALID_PROGRAM; @@ -588,7 +588,7 @@ hiprtcResult hiprtcGetProgramLog(hiprtcProgram p, char* l) return HIPRTC_SUCCESS; } -hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram p, std::size_t* sz) +extern "C" hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram p, std::size_t* sz) { if (!sz) return HIPRTC_ERROR_INVALID_INPUT; if (!isValidProgram(p)) return HIPRTC_ERROR_INVALID_PROGRAM; @@ -599,7 +599,7 @@ hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram p, std::size_t* sz) return HIPRTC_SUCCESS; } -hiprtcResult hiprtcGetCode(hiprtcProgram p, char* c) +extern "C" hiprtcResult hiprtcGetCode(hiprtcProgram p, char* c) { if (!c) return HIPRTC_ERROR_INVALID_INPUT; if (!isValidProgram(p)) return HIPRTC_ERROR_INVALID_PROGRAM; @@ -610,7 +610,7 @@ hiprtcResult hiprtcGetCode(hiprtcProgram p, char* c) return HIPRTC_SUCCESS; } -hiprtcResult hiprtcGetCodeSize(hiprtcProgram p, std::size_t* sz) +extern "C" hiprtcResult hiprtcGetCodeSize(hiprtcProgram p, std::size_t* sz) { if (!sz) return HIPRTC_ERROR_INVALID_INPUT; if (!isValidProgram(p)) return HIPRTC_ERROR_INVALID_PROGRAM; @@ -620,3 +620,15 @@ hiprtcResult hiprtcGetCodeSize(hiprtcProgram p, std::size_t* sz) return HIPRTC_SUCCESS; } + +extern "C" hiprtcResult hiprtcVersion(int* major, int* minor) +{ + if (major == nullptr || minor == nullptr) { + return HIPRTC_ERROR_INVALID_INPUT; + } + + *major = 9; + *minor = 0; + + return HIPRTC_SUCCESS; +} diff --git a/tests/src/hiprtc/hiprtcGetTypeName.cpp b/tests/src/hiprtc/hiprtcGetTypeName.cpp deleted file mode 100644 index 812229f81f..0000000000 --- a/tests/src/hiprtc/hiprtcGetTypeName.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/* -Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ -/* HIT_START - * BUILD: %t %s ../test_common.cpp LINK_OPTIONS hiprtc EXCLUDE_HIP_PLATFORM nvcc - * TEST: %t - * HIT_END - */ - -#include - -#define HIPRTC_GET_TYPE_NAME -#include -#include - -#include -#include -#include - -static constexpr auto gpu_program{ -R"( -#include - -namespace N1 { struct S1_t { int i; double d; }; } -template -__global__ void f3(int *result) { *result = sizeof(T); } -)"}; - -// note: this structure is also defined in GPU code string. Should ideally -// be in a header file included by both GPU code string and by CPU code. -namespace N1 { struct S1_t { int i; double d; }; }; - -template -std::string getKernelNameForType(void) -{ - std::string type_name; - hiprtcGetTypeName(&type_name); - return std::string{"f3<"} + type_name + '>'; -} - -int main() -{ - using namespace std; - - hiprtcProgram prog; - hiprtcCreateProgram(&prog, gpu_program, "gpu_program.cu", 0, nullptr, - nullptr); - - vector name_vec; - vector expected_result; - - name_vec.push_back(getKernelNameForType()); - expected_result.push_back(sizeof(int)); - name_vec.push_back(getKernelNameForType()); - expected_result.push_back(sizeof(double)); - name_vec.push_back(getKernelNameForType()); - expected_result.push_back(sizeof(N1::S1_t)); - - for (auto&& x : name_vec) hiprtcAddNameExpression(prog, x.c_str()); - - hipDeviceProp_t props; - int device = 0; - hipGetDeviceProperties(&props, device); - std::string gfxName = "gfx" + std::to_string(props.gcnArch); - std::string sarg = "--gpu-architecture=" + gfxName; - const char* options[] = { - sarg.c_str() - }; - - hiprtcResult compileResult = hiprtcCompileProgram(prog, 1, options); - - size_t logSize; - hiprtcGetProgramLogSize(prog, &logSize); - - if (logSize) { - string log(logSize, '\0'); - hiprtcGetProgramLog(prog, &log[0]); - - cout << log << '\n'; - } - - if (compileResult != HIPRTC_SUCCESS) { failed("Compilation failed."); } - - size_t codeSize; - hiprtcGetCodeSize(prog, &codeSize); - - vector code(codeSize); - hiprtcGetCode(prog, code.data()); - - hipModule_t module; - hipModuleLoadDataEx(&module, code.data(), 0, nullptr, nullptr); - - hipDeviceptr_t dResult; - int hResult = 0; - hipMalloc(&dResult, sizeof(hResult)); - hipMemcpyHtoD(dResult, &hResult, sizeof(hResult)); - - for (size_t i = 0; i < name_vec.size(); ++i) { - const char *name; - hiprtcGetLoweredName(prog, name_vec[i].c_str(), &name); - - hipFunction_t kernel; - hipModuleGetFunction(&kernel, module, name); - - struct { hipDeviceptr_t a_; } args{dResult}; - - auto size = sizeof(args); - void* config[] = {HIP_LAUNCH_PARAM_BUFFER_POINTER, &args, - HIP_LAUNCH_PARAM_BUFFER_SIZE, &size, - HIP_LAUNCH_PARAM_END}; - - hipModuleLaunchKernel(kernel, - 1, 1, 1, - 1, 1, 1, - 0, nullptr, - nullptr, config); - - hipMemcpyDtoH(&hResult, dResult, sizeof(hResult)); - - if (expected_result[i] != hResult) { failed("Validation failed."); } - } - - hipFree(dResult); - hipModuleUnload(module); - - hiprtcDestroyProgram(&prog); - - passed(); -} From 57edf481912bbb729838b8990546e9ad738b78cc Mon Sep 17 00:00:00 2001 From: Siu Chi Chan Date: Fri, 28 Feb 2020 06:17:41 -0500 Subject: [PATCH 061/154] improve code object loading error message (#1889) --- src/program_state.inl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/program_state.inl b/src/program_state.inl index bcac4d384b..27fa9f2b0e 100644 --- a/src/program_state.inl +++ b/src/program_state.inl @@ -426,7 +426,11 @@ public: auto check_hsa_error = [](hsa_status_t s) { if (s != HSA_STATUS_SUCCESS) { - hip_throw(std::runtime_error{"error when loading code object"}); + const char* hsa_err_msg; + hsa_status_string(s, &hsa_err_msg); + hip_throw(std::runtime_error{ + std::string("error when loading code object: ") + + hsa_err_msg}); } }; From d29ad504640ddf608368c0a97de5ef28a18b5fa0 Mon Sep 17 00:00:00 2001 From: Jatin Chaudhary <51944368+cjatin@users.noreply.github.com> Date: Fri, 28 Feb 2020 16:48:15 +0530 Subject: [PATCH 062/154] [dtests] __shfl_up and __shfl_down tests (#1899) --- tests/src/kernel/hipShflUpDownTest.cpp | 102 +++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/src/kernel/hipShflUpDownTest.cpp diff --git a/tests/src/kernel/hipShflUpDownTest.cpp b/tests/src/kernel/hipShflUpDownTest.cpp new file mode 100644 index 0000000000..553087ce45 --- /dev/null +++ b/tests/src/kernel/hipShflUpDownTest.cpp @@ -0,0 +1,102 @@ +/* +Copyright (c) 2015-present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +/* HIT_START + * BUILD: %t %s ../test_common.cpp + * TEST: %t + * HIT_END + */ + +#include +#include +#include "test_common.h" + +template +__global__ void shflDownSum(T* a, int size) { + T val = a[threadIdx.x]; + for (int i = size / 2; i > 0; i /= 2) { + val += __shfl_down(val, i, size); + } + a[threadIdx.x] = val; +} + +template +__global__ void shflUpSum(T* a, int size) { + T val = a[threadIdx.x]; + for (int i = size / 2; i > 0; i /= 2) { + val += __shfl_up(val, i, size); + } + a[threadIdx.x] = val; +} + +template +void runTestShflUp() { + const int size = 32; + T a[size]; + T cpuSum = 0; + for (int i = 0; i < size; i++) { + a[i] = i; + cpuSum += a[i]; + } + T* d_a; + hipMalloc(&d_a, sizeof(T) * size); + hipMemcpy(d_a, &a, sizeof(T) * size, hipMemcpyDefault); + hipLaunchKernelGGL(shflUpSum, 1, size, 0, 0, d_a, size); + hipMemcpy(&a, d_a, sizeof(T) * size, hipMemcpyDefault); + if (a[size - 1] != cpuSum) { + hipFree(d_a); + failed("Shfl Up Sum did not match."); + } + hipFree(d_a); +} + +template +void runTestShflDown() { + const int size = 32; + T a[size]; + T cpuSum = 0; + for (int i = 0; i < size; i++) { + a[i] = i; + cpuSum += a[i]; + } + T* d_a; + hipMalloc(&d_a, sizeof(T) * size); + hipMemcpy(d_a, &a, sizeof(T) * size, hipMemcpyDefault); + hipLaunchKernelGGL(shflDownSum, 1, size, 0, 0, d_a, size); + hipMemcpy(&a, d_a, sizeof(T) * size, hipMemcpyDefault); + if (a[0] != cpuSum) { + hipFree(d_a); + failed("Shfl Up Sum did not match."); + } + hipFree(d_a); +} +int main() { + runTestShflUp(); + runTestShflUp(); + runTestShflUp(); + runTestShflUp(); + + runTestShflDown(); + runTestShflDown(); + runTestShflDown(); + runTestShflDown(); + passed(); +} From e638f9afba924c961711a91fd87a92fc10facd41 Mon Sep 17 00:00:00 2001 From: Payam Date: Fri, 28 Feb 2020 17:54:29 -0500 Subject: [PATCH 063/154] updated hip-targets for VDI Change-Id: I268f1dbf04e3439ca1f3dd1cfbbc2f40667c00f9 --- packaging/hip-targets-release.cmake | 31 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/packaging/hip-targets-release.cmake b/packaging/hip-targets-release.cmake index 942f4f3be7..90fd961184 100644 --- a/packaging/hip-targets-release.cmake +++ b/packaging/hip-targets-release.cmake @@ -28,32 +28,31 @@ if(NOT HIP_RUNTIME MATCHES "VDI") list(APPEND _IMPORT_CHECK_TARGETS hip::hip_hcc_static ) list(APPEND _IMPORT_CHECK_FILES_FOR_hip::hip_hcc_static "${_IMPORT_PREFIX}/lib/libhip_hcc_static.a" ) -endif() -# Import target "hip::hip_hcc" for configuration "Release" -set_property(TARGET hip::hip_hcc APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) -if(HIP_COMPILER STREQUAL "clang") - if(HIP_RUNTIME MATCHES "VDI") - set_target_properties(hip::amdhip64 PROPERTIES - IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libamdhip64.so" - IMPORTED_SONAME_RELEASE "libamdhip64.so") + # Import target "hip::hip_hcc" for configuration "Release" + set_property(TARGET hip::hip_hcc APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) + if(HIP_COMPILER STREQUAL "clang") + set_target_properties(hip::hip_hcc PROPERTIES + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libhip_hcc.so" + IMPORTED_SONAME_RELEASE "libhip_hcc.so") else() set_target_properties(hip::hip_hcc PROPERTIES + IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE "hcc::hccrt;hcc::hc_am" IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libhip_hcc.so" IMPORTED_SONAME_RELEASE "libhip_hcc.so") endif() -else() - set_target_properties(hip::hip_hcc PROPERTIES - IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE "hcc::hccrt;hcc::hc_am" - IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libhip_hcc.so" - IMPORTED_SONAME_RELEASE "libhip_hcc.so") -endif() - -if(NOT HIP_RUNTIME MATCHES "VDI") list(APPEND _IMPORT_CHECK_TARGETS hip::hip_hcc ) list(APPEND _IMPORT_CHECK_FILES_FOR_hip::hip_hcc "${_IMPORT_PREFIX}/lib/libhip_hcc.so" ) + else() + + set_property(TARGET hip::amdhip64 APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) + set_target_properties(hip::amdhip64 PROPERTIES + IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libamdhip64.so" + IMPORTED_SONAME_RELEASE "libamdhip64.so") list(APPEND _IMPORT_CHECK_TARGETS hip::amdhip64) list(APPEND _IMPORT_CHECK_FILES_FOR_hip::amdhip64 "${_IMPORT_PREFIX}/lib/libamdhip64.so" ) + endif() + # Commands beyond this point should not need to know the version. set(CMAKE_IMPORT_FILE_VERSION) From 5b14ab34f5675066e0d167aa04f0a375a23f3687 Mon Sep 17 00:00:00 2001 From: Payam Date: Fri, 28 Feb 2020 18:13:16 -0500 Subject: [PATCH 064/154] updating hip debug target for VDI Change-Id: Ia08af89e2ee461e75e888e495984d3b1d9a6a858 --- packaging/hip-targets.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/hip-targets.cmake b/packaging/hip-targets.cmake index 90a76bef0f..ac72419f52 100644 --- a/packaging/hip-targets.cmake +++ b/packaging/hip-targets.cmake @@ -17,7 +17,7 @@ set(_targetsDefined) set(_targetsNotDefined) set(_expectedTargets) if(HIP_RUNTIME MATCHES "VDI") - foreach(_expectedTarget hip::hip_hcc hip::host hip::device) + foreach(_expectedTarget hip::amdhip64 hip::host hip::device) else() foreach(_expectedTarget hip:hip_hcc_static hip::hip_hcc hip::host hip::device) endif() From 87178f94563be66ebaa9dddc6ff77f29aec1a77b Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Sun, 1 Mar 2020 13:40:14 -0500 Subject: [PATCH 065/154] Fix hipMemcpy3d (partially) Incoming changes from upstream split the struct hipMemcpy3DParms into two separate ones - hipMemcpy3DParms and HIP_MEMCPY3D, which are cudaMemcpy3DParms and CUDA_MEMCPY3D equivalents respectively. Note that HIP_MEMCPY3D is missing half the members of CUDA_MEMCPY3D (this should be fixed in PR#1887). Work around this by using a substitute _HIP_MEMCPY3D struct for now. Change-Id: Ic15134e6deb260189b662b3804d2309a9b8473e9 --- vdi/hip_conversions.hpp | 34 ++++++++++++++- vdi/hip_memory.cpp | 91 ++++++++++++++++++++++++++--------------- 2 files changed, 91 insertions(+), 34 deletions(-) diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index 321ef7e024..b34680ec72 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -25,6 +25,36 @@ THE SOFTWARE. #include #include +// HIP_MEMCPY3D is currently broken. +// TODO remove this struct once the headers will be fixed. +struct _HIP_MEMCPY3D { + unsigned int srcXInBytes; + unsigned int srcY; + unsigned int srcZ; + unsigned int srcLOD; + hipMemoryType srcMemoryType; + const void* srcHost; + hipDeviceptr_t srcDevice; + hipArray_t srcArray; + unsigned int srcPitch; + unsigned int srcHeight; + + unsigned int dstXInBytes; + unsigned int dstY; + unsigned int dstZ; + unsigned int dstLOD; + hipMemoryType dstMemoryType; + void* dstHost; + hipDeviceptr_t dstDevice; + hipArray_t dstArray; + unsigned int dstPitch; + unsigned int dstHeight; + + unsigned int WidthInBytes; + unsigned int Height; + unsigned int Depth; +}; + namespace hip { inline @@ -549,8 +579,8 @@ std::pair getMemoryType(const hipMemcpyKind kind) } inline -hipMemcpy3DParms getMemcpy3DParms(const hip_Memcpy2D& desc2D) { - hipMemcpy3DParms desc3D = {}; +_HIP_MEMCPY3D getMemcpy3DParms(const hip_Memcpy2D& desc2D) { + _HIP_MEMCPY3D desc3D = {}; desc3D.srcXInBytes = desc2D.srcXInBytes; desc3D.srcY = desc2D.srcY; diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 6d68cfd9f5..7b4fece3b2 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -1324,7 +1324,7 @@ hipError_t ihipMemcpyAtoH(hipArray* srcArray, return hipSuccess; } -hipError_t ihipMemcpyParam3D(const hipMemcpy3DParms* pCopy, +hipError_t ihipMemcpyParam3D(const _HIP_MEMCPY3D* pCopy, hipStream_t stream, bool isAsync = false) { // If {src/dst}MemoryType is hipMemoryTypeUnified, {src/dst}Device and {src/dst}Pitch specify the (unified virtual address space) @@ -1379,13 +1379,13 @@ hipError_t ihipMemcpyParam3D(const hipMemcpy3DParms* pCopy, hipError_t ihipMemcpyParam2D(const hip_Memcpy2D* pCopy, hipStream_t stream, bool isAsync = false) { - hipMemcpy3DParms desc = hip::getMemcpy3DParms(*pCopy); + _HIP_MEMCPY3D desc = hip::getMemcpy3DParms(*pCopy); return ihipMemcpyParam3D(&desc, stream, isAsync); } hipError_t ihipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, - size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { + size_t height, hipMemcpyKind kind, hipStream_t stream, bool isAsync = false) { hip_Memcpy2D desc = {}; desc.srcXInBytes = 0; @@ -1537,55 +1537,70 @@ hipError_t hipMemcpyAtoH(void* dstHost, hipError_t ihipMemcpy3D(const hipMemcpy3DParms* p, hipStream_t stream, bool isAsync = false) { - // Having src and dst be an array is ambigous, since we can't tell - // if the user intended to call runtime or driver version of hipMemcpy3D(). - // For now hope that we never encounter this case. - assert((p->srcArray == nullptr) || (p->dstArray == nullptr)); + // The struct passed to hipMemcpy3D() must specify one of srcArray or srcPtr and one of dstArray or dstPtr. + // Passing more than one non-zero source or destination will cause hipMemcpy3D() to return an error. + if (((p->srcArray != nullptr) && (p->srcPtr.ptr != nullptr)) || + ((p->dstArray != nullptr) && (p->dstPtr.ptr != nullptr))) { + return hipErrorInvalidValue; + } - // Now we need to patch the user provided struct if they intended on calling the runtime version. - hipMemcpy3DParms pCopy = {}; - std::memcpy(&pCopy, p, sizeof(hipMemcpy3DParms)); + // If the source and destination are both arrays, hipMemcpy3D() will return an error if they do not have the same element size. + if (((p->srcArray != nullptr) && (p->dstArray != nullptr)) && + (hip::getElementSize(p->dstArray->Format) != hip::getElementSize(p->dstArray->Format))) { + return hipErrorInvalidValue; + } - if ((p->srcPtr.ptr != nullptr) || (p->dstPtr.ptr != nullptr)) { - pCopy.WidthInBytes = p->extent.width; - pCopy.Height = p->extent.height; - pCopy.Depth = p->extent.depth; + _HIP_MEMCPY3D pCopy = {}; + + pCopy.WidthInBytes = p->extent.width; + pCopy.Height = p->extent.height; + pCopy.Depth = p->extent.depth; + + pCopy.srcXInBytes = p->srcPos.x; + pCopy.srcY = p->srcPos.y; + pCopy.srcZ = p->srcPos.z; + pCopy.srcLOD = 0; + + pCopy.dstXInBytes = p->dstPos.x; + pCopy.dstY = p->dstPos.y; + pCopy.dstZ = p->dstPos.z; + pCopy.dstLOD = 0; + + if (p->srcArray != nullptr) { + pCopy.srcMemoryType = hipMemoryTypeArray; + // When reffering to array memory, hipPos::x is in elements. + pCopy.srcXInBytes *= hip::getElementSize(p->dstArray->Format); } if (p->srcPtr.ptr != nullptr) { - pCopy.srcXInBytes = p->srcPos.x; - pCopy.srcY = p->srcPos.y; - pCopy.srcZ = p->srcPos.z; - pCopy.srcLOD = 0; pCopy.srcMemoryType = std::get<0>(hip::getMemoryType(p->kind)); pCopy.srcHost = p->srcPtr.ptr; pCopy.srcDevice = p->srcPtr.ptr; pCopy.srcPitch = p->srcPtr.pitch; pCopy.srcHeight = p->srcPtr.ysize; + } - if (p->dstArray != nullptr) { - pCopy.dstMemoryType = hipMemoryTypeArray; - // When reffering to array memory, hipExtent::width is in elements. - pCopy.WidthInBytes *= hip::getElementSize(p->dstArray->Format); - } + if (p->dstArray != nullptr) { + pCopy.dstMemoryType = hipMemoryTypeArray; + // When reffering to array memory, hipPos::x is in elements. + pCopy.srcXInBytes *= hip::getElementSize(p->dstArray->Format); } if (p->dstPtr.ptr != nullptr) { - pCopy.dstXInBytes = p->dstPos.x; - pCopy.dstY = p->dstPos.y; - pCopy.dstZ = p->dstPos.z; - pCopy.dstLOD = 0; pCopy.dstMemoryType = std::get<1>(hip::getMemoryType(p->kind)); pCopy.dstHost = p->dstPtr.ptr; pCopy.dstDevice = p->dstPtr.ptr; pCopy.dstPitch = p->dstPtr.pitch; pCopy.dstHeight = p->dstPtr.ysize; + } - if (p->srcArray != nullptr) { - pCopy.srcMemoryType = hipMemoryTypeArray; - // When reffering to array memory, hipExtent::width is in elements. - pCopy.WidthInBytes *= hip::getElementSize(p->srcArray->Format); - } + // If a HIP array is participating in the copy, the extent is defined in terms of that array's elements. + if ((p->srcArray != nullptr) && (p->dstArray == nullptr)) { + pCopy.WidthInBytes *= hip::getElementSize(p->srcArray->Format); + } else if ((p->srcArray == nullptr) && (p->dstArray != nullptr)) { + pCopy.WidthInBytes *= hip::getElementSize(p->dstArray->Format); + } else if ((p->srcArray != nullptr) && (p->dstArray != nullptr)) { + pCopy.WidthInBytes *= hip::getElementSize(p->dstArray->Format); } return ihipMemcpyParam3D(&pCopy, stream, isAsync); @@ -1603,6 +1618,18 @@ hipError_t hipMemcpy3DAsync(const hipMemcpy3DParms* p, hipStream_t stream) { HIP_RETURN(ihipMemcpy3D(p, stream, true)); } +hipError_t hipDrvMemcpy3D(const _HIP_MEMCPY3D* pCopy) { + HIP_INIT_API(hipDrvMemcpy3D, pCopy); + + HIP_RETURN(ihipMemcpyParam3D(pCopy, nullptr)); +} + +hipError_t hipDrvMemcpy3DAsync(const _HIP_MEMCPY3D* pCopy, hipStream_t stream) { + HIP_INIT_API(hipDrvMemcpy3DAsync, pCopy, stream); + + HIP_RETURN(ihipMemcpyParam3D(pCopy, stream, true)); +} + hipError_t ihipMemset(void* dst, int value, size_t valueSize, size_t sizeBytes, hipStream_t stream, bool isAsync = false) { if (sizeBytes == 0) { From fe1fad4805499b258185b87eee40f0c39e5b426f Mon Sep 17 00:00:00 2001 From: German Andryeyev Date: Mon, 2 Mar 2020 18:35:56 -0500 Subject: [PATCH 066/154] SWDEV-193956 - [hipclang-vdi-rocm][perf] ~45% to 50% of Performance drop on rocBLAS_int8 test Enable cudaSetDeviceFlags() api call. Use active wait by default for all devices. Change-Id: Ifc2ebe3dd9b0aa3fdbfbc9cb5c2cd8b3b726124f --- vdi/hip_context.cpp | 3 +++ vdi/hip_device_runtime.cpp | 28 +++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/vdi/hip_context.cpp b/vdi/hip_context.cpp index f28ca48239..6151f68c6a 100644 --- a/vdi/hip_context.cpp +++ b/vdi/hip_context.cpp @@ -50,6 +50,9 @@ void init() { amd::Context* context = new amd::Context(device, amd::Context::Info()); if (!context) return; + // Enable active wait on the device by default + devices[i]->SetActiveWait(true); + if (context && CL_SUCCESS != context->create(nullptr)) { context->release(); } else { diff --git a/vdi/hip_device_runtime.cpp b/vdi/hip_device_runtime.cpp index be2093a8c8..acf73fa656 100644 --- a/vdi/hip_device_runtime.cpp +++ b/vdi/hip_device_runtime.cpp @@ -491,16 +491,34 @@ hipError_t hipSetDevice ( int device ) { hipError_t hipSetDeviceFlags ( unsigned int flags ) { HIP_INIT_API(hipSetDeviceFlags, flags); - /* FIXME */ - /* Not all of Ctx may be implemented */ - - unsigned supportedFlags = + constexpr uint32_t supportedFlags = hipDeviceScheduleMask | hipDeviceMapHost | hipDeviceLmemResizeToMax; - if (flags & (~supportedFlags)) { + if (flags & ~supportedFlags) { HIP_RETURN(hipErrorInvalidValue); } + amd::Device* device = hip::getCurrentDevice()->devices()[0]; + switch (flags & hipDeviceScheduleMask) { + case hipDeviceScheduleAuto: + // Current behavior is different from the spec, due to MT usage in runtime + if (hip::host_device->devices().size() >= std::thread::hardware_concurrency()) { + device->SetActiveWait(false); + break; + } + // Fall through for active wait... + case hipDeviceScheduleSpin: + case hipDeviceScheduleYield: + // The both options falls into yield, because MT usage in runtime + device->SetActiveWait(true); + break; + case hipDeviceScheduleBlockingSync: + device->SetActiveWait(false); + break; + default: + break; + } + HIP_RETURN(hipSuccess); } From 1561f6164246197dd8ea21c1af5b261283e69677 Mon Sep 17 00:00:00 2001 From: Evgeny Mankov Date: Tue, 3 Mar 2020 12:07:13 +0300 Subject: [PATCH 067/154] [HIP][cmake] Remove dependency from hipify-clang [Reason] Upcoming hipify-clang's splitting out into a new repository https://github.com/ROCm-Developer-Tools/HIPIFY. --- CMakeLists.txt | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 07d23130c2..4894168348 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,11 +1,6 @@ cmake_minimum_required(VERSION 3.4.3) project(hip) -############################# -# Options -############################# -option(BUILD_HIPIFY_CLANG "Enable building the CUDA->HIP converter" OFF) - ############################# # Setup config generation ############################# @@ -238,11 +233,6 @@ set(LIB_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib) set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/include) set(CONFIG_PACKAGE_INSTALL_DIR ${LIB_INSTALL_DIR}/cmake/hip) -# Build clang hipify if enabled -if (BUILD_HIPIFY_CLANG) - add_subdirectory(hipify-clang) -endif() - # Build LPL an CA (fat binary generation / fat binary decomposition tools) if # platform is hcc; do this before the ugly hijacking of the compiler, since no # HC code is involved. @@ -464,11 +454,6 @@ add_custom_target(pkg_hip_base COMMAND ${CMAKE_COMMAND} . WORKING_DIRECTORY ${BUILD_DIR} DEPENDS lpl ca) -# Packaging needs to wait for hipify-clang to build if it's enabled... -if (BUILD_HIPIFY_CLANG) - add_dependencies(pkg_hip_base hipify-clang) -endif() - # Package: hip_hcc set(BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/packages/hip_hcc) configure_file(packaging/hip-hcc.txt ${BUILD_DIR}/CMakeLists.txt @ONLY) From b910a7ff263a052165a643fa99f82cecd78d4dde Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Mon, 2 Mar 2020 14:58:33 -0500 Subject: [PATCH 068/154] SWDEV-219799 Add missing __hip_pinned_shadow__ attribute to the hipModuleTexture2dDrv test. Change-Id: I2c213a4108611c1af691388f268cac429b81a313 --- tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp | 5 ++++- tests/src/runtimeApi/module/tex2d_kernel.cpp | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp b/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp index 3903acd125..d4c5c31283 100644 --- a/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp +++ b/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ @@ -33,6 +33,9 @@ THE SOFTWARE. #define fileName "tex2d_kernel.code" +#if __HIP__ +__hip_pinned_shadow__ +#endif texture tex; bool testResult = false; diff --git a/tests/src/runtimeApi/module/tex2d_kernel.cpp b/tests/src/runtimeApi/module/tex2d_kernel.cpp index e66d690907..9a22ed25ed 100644 --- a/tests/src/runtimeApi/module/tex2d_kernel.cpp +++ b/tests/src/runtimeApi/module/tex2d_kernel.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD_CMD: tex2d_kernel.code %hc --genco %S/tex2d_kernel.cpp -o tex2d_kernel.code EXCLUDE_HIP_PLATFORM vdi + * BUILD_CMD: tex2d_kernel.code %hc --genco %S/tex2d_kernel.cpp -o directed_tests/runtimeApi/module/tex2d_kernel.code * HIT_END */ From 25f107e2b6a75efe1ff76617efe708ff3d63ec3e Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 3 Mar 2020 16:14:12 -0500 Subject: [PATCH 069/154] Fix Windows build. Calm down warning C4715: not all control paths return a value. Change-Id: I99d939b84770a499bb6a0edcc3bc0bf12961a711 --- vdi/hip_conversions.hpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index b34680ec72..de3cbaad0a 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -101,6 +101,8 @@ cl_channel_type getCLChannelType(const hipArray_Format hipFormat, } ShouldNotReachHere(); + + return {}; } inline @@ -115,6 +117,8 @@ cl_channel_order getCLChannelOrder(const unsigned int hipNumChannels) { } ShouldNotReachHere(); + + return {}; } inline @@ -139,6 +143,8 @@ cl_mem_object_type getCLMemObjectType(const unsigned int hipWidth, } ShouldNotReachHere(); + + return {}; } inline @@ -155,6 +161,8 @@ cl_addressing_mode getCLAddressingMode(const hipTextureAddressMode hipAddressMod } ShouldNotReachHere(); + + return {}; } inline @@ -167,6 +175,8 @@ cl_filter_mode getCLFilterMode(const hipTextureFilterMode hipFilterMode) { } ShouldNotReachHere(); + + return {}; } inline @@ -179,6 +189,8 @@ cl_mem_object_type getCLMemObjectType(const hipResourceType hipResType) { } ShouldNotReachHere(); + + return {}; } inline @@ -198,6 +210,8 @@ size_t getElementSize(const hipArray_Format arrayFormat) { } ShouldNotReachHere(); + + return {}; } inline @@ -279,6 +293,8 @@ hipChannelFormatDesc getChannelFormatDesc(int numChannels, } ShouldNotReachHere(); + + return {}; } inline @@ -317,6 +333,8 @@ hipArray_Format getArrayFormat(const hipChannelFormatDesc& desc) { } ShouldNotReachHere(); + + return {}; } inline @@ -349,6 +367,8 @@ int getNumChannels(const hipResourceViewFormat hipFormat) { } ShouldNotReachHere(); + + return {}; } inline @@ -389,6 +409,8 @@ hipArray_Format getArrayFormat(const hipResourceViewFormat hipFormat) { } ShouldNotReachHere(); + + return {}; } inline @@ -481,6 +503,8 @@ hipResourceViewFormat getResourceViewFormat(const hipChannelFormatDesc& desc) { } ShouldNotReachHere(); + + return {}; } inline @@ -576,6 +600,8 @@ std::pair getMemoryType(const hipMemcpyKind kind) } ShouldNotReachHere(); + + return {}; } inline From 3479847d16eba2dd0bc9dd654eb881a8b0adac6b Mon Sep 17 00:00:00 2001 From: agodavar Date: Wed, 4 Mar 2020 20:23:06 +0530 Subject: [PATCH 070/154] Enable tests hipTestDeviceSymbol,hipTestConstant and p2p_copy_coherency on hip-vdi SWDEV-225266: [HIP-VDI] HIP-VDI disabled tests (p2p_copy_coherency.cpp) SWDEV-225388: hipTestDeviceSymbol.cpp & hipTestConstant.cpp failed to build on hip-vdi For hipTestDeviceSymbol.cpp & hipTestConstant.cpp tests: Currently "__HIP_VDI__" flag is enabled in CMakeLists.txt, but when application is compiled with hipcc, __HIP_VDI__ is not defined to differentiate if compiled for VDI/HCC for headers. For ./src/runtimeApi/memory/p2p_copy_coherency.cpp: Fixed compilation issue to include only when compile for HCC runtime " not found" Currently test is disabled to run on all platforms. When validated on multi-GPU machine, memcpy between multiple GPUs via GPU synchronization is not working on hcc and vdi path. Need to validate on nvidia machine to know if test is valid. Disabled GPU synchronization test for now. For ./src/runtimeApi/module/hipModuleTexture2dDrv.cpp: updated test to generate tex2d_kernel.code object in build directory. Currently ctest looks for it in build directory. Change-Id: I629d395a919c2440d921422716944c7940ed6010 --- bin/hipcc | 4 ++-- tests/src/deviceLib/hipTestDeviceSymbol.cpp | 2 +- tests/src/kernel/hipTestConstant.cpp | 2 +- tests/src/runtimeApi/memory/p2p_copy_coherency.cpp | 14 ++++++++------ tests/src/runtimeApi/module/tex2d_kernel.cpp | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/bin/hipcc b/bin/hipcc index 662bdd730e..7eacc8f6b6 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -221,8 +221,8 @@ if ($HIP_PLATFORM eq "clang") { $HSA_PATH=$ENV{'HSA_PATH'} // "$ROCM_PATH/hsa"; $HIPCXXFLAGS .= " -isystem $HSA_PATH/include"; - if (! $HIP_RUNTIME eq "HCC" ) { - $HIPCXXFLAGS .= " -fhip-new-launch-api"; + if (!($HIP_RUNTIME eq "HCC")) { + $HIPCXXFLAGS .= " -D__HIP_VDI__ -fhip-new-launch-api"; } } elsif ($HIP_PLATFORM eq "hcc") { diff --git a/tests/src/deviceLib/hipTestDeviceSymbol.cpp b/tests/src/deviceLib/hipTestDeviceSymbol.cpp index 56b2d9c8e1..4c2dd279ea 100644 --- a/tests/src/deviceLib/hipTestDeviceSymbol.cpp +++ b/tests/src/deviceLib/hipTestDeviceSymbol.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s * TEST: %t * HIT_END */ diff --git a/tests/src/kernel/hipTestConstant.cpp b/tests/src/kernel/hipTestConstant.cpp index 5dda5d5b0a..575e28966f 100644 --- a/tests/src/kernel/hipTestConstant.cpp +++ b/tests/src/kernel/hipTestConstant.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp * TEST: %t * HIT_END */ diff --git a/tests/src/runtimeApi/memory/p2p_copy_coherency.cpp b/tests/src/runtimeApi/memory/p2p_copy_coherency.cpp index d02053d247..f459a9d100 100644 --- a/tests/src/runtimeApi/memory/p2p_copy_coherency.cpp +++ b/tests/src/runtimeApi/memory/p2p_copy_coherency.cpp @@ -23,7 +23,7 @@ THE SOFTWARE. // Also serves as a template for other tests. /* HIT_START - * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS --std=c++11 EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS --std=c++11 * TEST: %t EXCLUDE_HIP_PLATFORM all * HIT_END @@ -32,12 +32,13 @@ THE SOFTWARE. #include "hip/hip_runtime.h" #include "test_common.h" -#ifdef __HIP_PLATFORM_HCC__ -#include -#endif - #define USE_HCC_MEMTRACKER 0 /* Debug flag to show the memtracker periodically */ +#ifndef __HIP_VDI__ +#include +#else +#define USE_HCC_MEMTRACKER 0 +#endif int elementSizes[] = {1, 16, 1024, 524288, 16 * 1000 * 1000}; int nSizes = sizeof(elementSizes) / sizeof(int); @@ -202,7 +203,8 @@ int main(int argc, char* argv[]) { }; for (int index = 0; index < nSizes; index++) { - testMultiGpu(dev0, dev1, elementSizes[index], false /*GPU Synchronization*/); + //ToDo: Enable when verified on all platforms + //testMultiGpu(dev0, dev1, elementSizes[index], false /*GPU Synchronization*/); testMultiGpu(dev0, dev1, elementSizes[index], true /*Host Synchronization*/); } diff --git a/tests/src/runtimeApi/module/tex2d_kernel.cpp b/tests/src/runtimeApi/module/tex2d_kernel.cpp index 9a22ed25ed..be121f3e69 100644 --- a/tests/src/runtimeApi/module/tex2d_kernel.cpp +++ b/tests/src/runtimeApi/module/tex2d_kernel.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD_CMD: tex2d_kernel.code %hc --genco %S/tex2d_kernel.cpp -o directed_tests/runtimeApi/module/tex2d_kernel.code + * BUILD_CMD: tex2d_kernel.code %hc --genco %S/tex2d_kernel.cpp -o tex2d_kernel.code * HIT_END */ From 3d64a389533c681def4da5f34f8f6ca8270a2481 Mon Sep 17 00:00:00 2001 From: Payam Date: Wed, 4 Mar 2020 13:49:58 -0500 Subject: [PATCH 071/154] reverting part of http://gerrit-git.amd.com/c/compute/ec/hip/+/326145 the test still fails Change-Id: I2e5968115a717e8482be670d9fb8472e391922d7 --- tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp b/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp index d4c5c31283..e7c254e9fd 100644 --- a/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp +++ b/tests/src/runtimeApi/module/hipModuleTexture2dDrv.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi * TEST: %t * HIT_END */ From 6a5d04209c79f9f9aed4c799ddf2d44e5931befe Mon Sep 17 00:00:00 2001 From: agodavar Date: Thu, 27 Feb 2020 20:06:10 +0530 Subject: [PATCH 072/154] Fix hipExtLaunchMultiKernelMultiDevice compilation issue Fix compilation error on hip-hcc+clang , hip-vdi+clang Enabled hipExtLaunchMultiKernelMultiDevice test on hip-vdi path hipExtLaunchMultiKernelMultiDevice common declaration for all paths Change-Id: I76031840614fce8e12a8e845548fa43a389a741a --- include/hip/hcc_detail/functional_grid_launch.hpp | 10 ---------- include/hip/hcc_detail/hip_runtime_api.h | 2 -- src/hip_module.cpp | 8 ++++++++ .../module/hipExtLaunchMultiKernelMultiDevice.cpp | 2 +- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/include/hip/hcc_detail/functional_grid_launch.hpp b/include/hip/hcc_detail/functional_grid_launch.hpp index 76a04fa355..cf4422070f 100644 --- a/include/hip/hcc_detail/functional_grid_launch.hpp +++ b/include/hip/hcc_detail/functional_grid_launch.hpp @@ -189,16 +189,6 @@ void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks, stream, &config[0]); } -inline -__attribute__((visibility("hidden"))) -hipError_t hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, - int numDevices, unsigned int flags) { - hip_impl::hip_init(); - auto& ps = hip_impl::get_program_state(); - return ihipExtLaunchMultiKernelMultiDevice(launchParamsList, numDevices, flags, ps); - -} - template inline __attribute__((visibility("hidden"))) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 0712db17f9..936170b1fd 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -2962,7 +2962,6 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor( hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( uint32_t* numBlocks, hipFunction_t f, uint32_t blockSize, size_t dynSharedMemPerBlk, unsigned int flags); -#if __HIP_VDI__ && !defined(__HCC__) /** * @brief Launches kernels on multiple devices and guarantees all specified kernels are dispatched * on respective streams before enqueuing any other work on the specified streams from any other threads @@ -2977,7 +2976,6 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( hipError_t hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, int numDevices, unsigned int flags); -#endif // doxygen end Version Management /** diff --git a/src/hip_module.cpp b/src/hip_module.cpp index 18b869cee7..8901f2c4aa 100644 --- a/src/hip_module.cpp +++ b/src/hip_module.cpp @@ -392,6 +392,14 @@ hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList return result; } +__attribute__((visibility("default"))) +hipError_t hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, + int numDevices, unsigned int flags) { + HIP_INIT_API(hipExtLaunchMultiKernelMultiDevice, launchParamsList, numDevices, flags); + auto& ps = hip_impl::get_program_state(); + return ihipExtLaunchMultiKernelMultiDevice(launchParamsList, numDevices, flags, ps); +} + namespace { // kernel for initializing GWS // nwm1 is the total number of work groups minus 1 diff --git a/tests/src/runtimeApi/module/hipExtLaunchMultiKernelMultiDevice.cpp b/tests/src/runtimeApi/module/hipExtLaunchMultiKernelMultiDevice.cpp index cc7a06d66b..fe9b06713c 100644 --- a/tests/src/runtimeApi/module/hipExtLaunchMultiKernelMultiDevice.cpp +++ b/tests/src/runtimeApi/module/hipExtLaunchMultiKernelMultiDevice.cpp @@ -23,7 +23,7 @@ THE SOFTWARE. // single GPU or multi GPUs. /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ From f70a66a7b34a4fb130f85ddf2e490e6c3e03358f Mon Sep 17 00:00:00 2001 From: Michael LIAO Date: Tue, 3 Mar 2020 16:36:37 -0500 Subject: [PATCH 073/154] Fix `-Werror=switch`. Change-Id: Ib0850a3f79308efeeecd18bb03ecd82bd355ef85 --- vdi/hip_conversions.hpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index de3cbaad0a..abaa4d1949 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -186,6 +186,8 @@ cl_mem_object_type getCLMemObjectType(const hipResourceType hipResType) { return CL_MEM_OBJECT_IMAGE1D; case hipResourceTypePitch2D: return CL_MEM_OBJECT_IMAGE2D; + default: + break; } ShouldNotReachHere(); @@ -330,6 +332,8 @@ hipArray_Format getArrayFormat(const hipChannelFormatDesc& desc) { case 32: return HIP_AD_FORMAT_FLOAT; } + default: + break; } ShouldNotReachHere(); @@ -364,6 +368,8 @@ int getNumChannels(const hipResourceViewFormat hipFormat) { case hipResViewFormatHalf4: case hipResViewFormatFloat4: return 4; + default: + break; } ShouldNotReachHere(); @@ -406,6 +412,8 @@ hipArray_Format getArrayFormat(const hipResourceViewFormat hipFormat) { case hipResViewFormatFloat2: case hipResViewFormatFloat4: return HIP_AD_FORMAT_FLOAT; + default: + break; } ShouldNotReachHere(); @@ -500,6 +508,8 @@ hipResourceViewFormat getResourceViewFormat(const hipChannelFormatDesc& desc) { return hipResViewFormatFloat4; } } + default: + break; } ShouldNotReachHere(); @@ -636,4 +646,4 @@ _HIP_MEMCPY3D getMemcpy3DParms(const hip_Memcpy2D& desc2D) { return desc3D; } -}; \ No newline at end of file +}; From 65ffaf82d86161be5ffd9eb37727a92b4a365c6f Mon Sep 17 00:00:00 2001 From: agodavar Date: Tue, 3 Mar 2020 13:33:52 +0530 Subject: [PATCH 074/154] Enable tests that are passing on hip-vdi Change-Id: I7de965f7e8bb7e4d0fa61cd584f0cd118c1e212e --- tests/src/deviceLib/hipSimpleAtomicsTest.cpp | 2 +- tests/src/deviceLib/hip_anyall.cpp | 2 +- tests/src/deviceLib/hip_ballot.cpp | 2 +- tests/src/deviceLib/hip_test_ldg.cpp | 2 +- tests/src/hipHcc.cpp | 2 +- tests/src/kernel/hipDynamicShared.cpp | 2 +- tests/src/kernel/hipDynamicShared2.cpp | 2 +- tests/src/kernel/hipExtLaunchKernelGGL.cpp | 2 +- tests/src/runtimeApi/memory/hipMemset2D.cpp | 2 +- tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp | 4 ++-- tests/src/texture/simpleTexture2DLayered.cpp | 4 ++-- vdi/hip_memory.cpp | 2 ++ 12 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/src/deviceLib/hipSimpleAtomicsTest.cpp b/tests/src/deviceLib/hipSimpleAtomicsTest.cpp index 029191aebf..760d65e555 100644 --- a/tests/src/deviceLib/hipSimpleAtomicsTest.cpp +++ b/tests/src/deviceLib/hipSimpleAtomicsTest.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp NVCC_OPTIONS -std=c++11 --gpu-architecture=sm_60 EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp NVCC_OPTIONS -std=c++11 --gpu-architecture=sm_60 * TEST: %t * HIT_END */ diff --git a/tests/src/deviceLib/hip_anyall.cpp b/tests/src/deviceLib/hip_anyall.cpp index aa27d106b7..c77a3cd25d 100644 --- a/tests/src/deviceLib/hip_anyall.cpp +++ b/tests/src/deviceLib/hip_anyall.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp NVCC_OPTIONS --Wno-deprecated-declarations EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp NVCC_OPTIONS --Wno-deprecated-declarations * TEST: %t * HIT_END */ diff --git a/tests/src/deviceLib/hip_ballot.cpp b/tests/src/deviceLib/hip_ballot.cpp index 3728575e80..324047a6b1 100644 --- a/tests/src/deviceLib/hip_ballot.cpp +++ b/tests/src/deviceLib/hip_ballot.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp NVCC_OPTIONS --Wno-deprecated-declarations EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp NVCC_OPTIONS --Wno-deprecated-declarations * TEST: %t * HIT_END */ diff --git a/tests/src/deviceLib/hip_test_ldg.cpp b/tests/src/deviceLib/hip_test_ldg.cpp index ccd87c4661..0f97c1f0ad 100644 --- a/tests/src/deviceLib/hip_test_ldg.cpp +++ b/tests/src/deviceLib/hip_test_ldg.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp NVCC_OPTIONS --gpu-architecture=sm_35 EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp NVCC_OPTIONS --gpu-architecture=sm_35 * TEST: %t * HIT_END */ diff --git a/tests/src/hipHcc.cpp b/tests/src/hipHcc.cpp index 5ebc2b820f..9bcc999b34 100644 --- a/tests/src/hipHcc.cpp +++ b/tests/src/hipHcc.cpp @@ -22,7 +22,7 @@ THE SOFTWARE. // Test the HCC-specific API extensions for HIP: /* HIT_START - * BUILD: %t %s EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s EXCLUDE_HIP_PLATFORM nvcc * TEST: %t EXCLUDE_HIP_PLATFORM all * HIT_END */ diff --git a/tests/src/kernel/hipDynamicShared.cpp b/tests/src/kernel/hipDynamicShared.cpp index 0da709f01c..d9a5942085 100644 --- a/tests/src/kernel/hipDynamicShared.cpp +++ b/tests/src/kernel/hipDynamicShared.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM hcc vdi + * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM hcc * TEST: %t EXCLUDE_HIP_PLATFORM nvcc * HIT_END */ diff --git a/tests/src/kernel/hipDynamicShared2.cpp b/tests/src/kernel/hipDynamicShared2.cpp index 2674f2eb84..11f0640537 100644 --- a/tests/src/kernel/hipDynamicShared2.cpp +++ b/tests/src/kernel/hipDynamicShared2.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM hcc vdi + * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM hcc * TEST: %t EXCLUDE_HIP_PLATFORM nvcc * HIT_END */ diff --git a/tests/src/kernel/hipExtLaunchKernelGGL.cpp b/tests/src/kernel/hipExtLaunchKernelGGL.cpp index 61a663b91c..39c660322b 100644 --- a/tests/src/kernel/hipExtLaunchKernelGGL.cpp +++ b/tests/src/kernel/hipExtLaunchKernelGGL.cpp @@ -19,7 +19,7 @@ THE SOFTWARE. // Test the Grid_Launch syntax. /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ diff --git a/tests/src/runtimeApi/memory/hipMemset2D.cpp b/tests/src/runtimeApi/memory/hipMemset2D.cpp index 83c73416a3..e7fe932b3d 100644 --- a/tests/src/runtimeApi/memory/hipMemset2D.cpp +++ b/tests/src/runtimeApi/memory/hipMemset2D.cpp @@ -23,7 +23,7 @@ THE SOFTWARE. // Also serves as a template for other tests. /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../../test_common.cpp * TEST: %t * HIT_END */ diff --git a/tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp b/tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp index 3c85a8afea..f4c72ca1c5 100755 --- a/tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp +++ b/tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp @@ -18,8 +18,8 @@ THE SOFTWARE. */ /* HIT_START - * BUILD_CMD: matmul.code %hc --genco %S/matmul.cpp -o matmul.code EXCLUDE_HIP_PLATFORM nvcc vdi - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD_CMD: matmul.code %hc --genco %S/matmul.cpp -o matmul.code EXCLUDE_HIP_PLATFORM nvcc + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ diff --git a/tests/src/texture/simpleTexture2DLayered.cpp b/tests/src/texture/simpleTexture2DLayered.cpp index f4d3aac1e5..e72ea3483a 100644 --- a/tests/src/texture/simpleTexture2DLayered.cpp +++ b/tests/src/texture/simpleTexture2DLayered.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp * TEST: %t * HIT_END */ @@ -66,7 +66,7 @@ void runTest(int width,int height,int num_layers,texturesrcArray != nullptr) { pCopy.srcMemoryType = hipMemoryTypeArray; + pCopy.srcArray = p->srcArray; // When reffering to array memory, hipPos::x is in elements. pCopy.srcXInBytes *= hip::getElementSize(p->dstArray->Format); } @@ -1582,6 +1583,7 @@ hipError_t ihipMemcpy3D(const hipMemcpy3DParms* p, if (p->dstArray != nullptr) { pCopy.dstMemoryType = hipMemoryTypeArray; + pCopy.dstArray = p->dstArray; // When reffering to array memory, hipPos::x is in elements. pCopy.srcXInBytes *= hip::getElementSize(p->dstArray->Format); } From f28e2ebb4225af53bafd333c68cc9d1e8542978e Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Thu, 5 Mar 2020 13:58:37 -0500 Subject: [PATCH 075/154] Fix typos in ihipMemcpy3D() Change-Id: I8720f113642f00bb013cf46284e9b13cc932bf4a --- vdi/hip_memory.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 33f3e150d3..07bbb7f71c 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -1570,7 +1570,7 @@ hipError_t ihipMemcpy3D(const hipMemcpy3DParms* p, pCopy.srcMemoryType = hipMemoryTypeArray; pCopy.srcArray = p->srcArray; // When reffering to array memory, hipPos::x is in elements. - pCopy.srcXInBytes *= hip::getElementSize(p->dstArray->Format); + pCopy.srcXInBytes *= hip::getElementSize(p->srcArray->Format); } if (p->srcPtr.ptr != nullptr) { @@ -1585,7 +1585,7 @@ hipError_t ihipMemcpy3D(const hipMemcpy3DParms* p, pCopy.dstMemoryType = hipMemoryTypeArray; pCopy.dstArray = p->dstArray; // When reffering to array memory, hipPos::x is in elements. - pCopy.srcXInBytes *= hip::getElementSize(p->dstArray->Format); + pCopy.dstXInBytes *= hip::getElementSize(p->dstArray->Format); } if (p->dstPtr.ptr != nullptr) { From 3fc50731ae2793cb8c93f2a5c2c0108eba7ebac1 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Thu, 5 Mar 2020 14:00:43 -0500 Subject: [PATCH 076/154] Correct logic in ihipMemcpyAtoA() HIP assumes that image width is in bytes, but VDI assumes that image width in pixels. Need to perform byte -> pixel coversion before doing anything. Change-Id: Ia9fd1f46d05db3fbe8049add10b4d7e5118a2b9a --- vdi/hip_memory.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 07bbb7f71c..cc3908ca64 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -1171,7 +1171,7 @@ hipError_t ihipMemcpyAtoA(hipArray* srcArray, hipArray* dstArray, amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D region, + amd::Coord3D copyRegion, hipStream_t stream, bool isAsync = false) { cl_mem srcMemObj = reinterpret_cast(srcArray->data); @@ -1183,8 +1183,16 @@ hipError_t ihipMemcpyAtoA(hipArray* srcArray, amd::Image* srcImage = as_amd(srcMemObj)->asImage(); amd::Image* dstImage = as_amd(dstMemObj)->asImage(); - if (!srcImage->validateRegion(srcOrigin, region) || - !dstImage->validateRegion(dstOrigin, region)) { + // HIP assumes the width is in bytes, but OCL assumes it's in pixels. + // Note that src and dst should have the same element size. + assert(srcImage->getImageFormat().getElementSize() == dstImage->getImageFormat().getElementSize()); + const size_t elementSize = srcImage->getImageFormat().getElementSize(); + static_cast(srcOrigin)[0] /= elementSize; + static_cast(dstOrigin)[0] /= elementSize; + static_cast(copyRegion)[0] /= elementSize; + + if (!srcImage->validateRegion(srcOrigin, copyRegion) || + !dstImage->validateRegion(dstOrigin, copyRegion)) { return hipErrorInvalidValue; } @@ -1195,7 +1203,7 @@ hipError_t ihipMemcpyAtoA(hipArray* srcArray, *dstImage, srcOrigin, dstOrigin, - region); + copyRegion); if (command == nullptr) { return hipErrorOutOfMemory; From c5aa54cbd24a7aa3cdc9265ea639d55f4b457918 Mon Sep 17 00:00:00 2001 From: jujiang Date: Thu, 5 Mar 2020 16:50:27 -0500 Subject: [PATCH 077/154] SWDEV-225338-simpletexture3D failed-Fix bug in test app Change-Id: I106d1951368e0c7a8ccd6aa462c22f81e8332cdc --- tests/src/texture/simpleTexture3D.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/src/texture/simpleTexture3D.cpp b/tests/src/texture/simpleTexture3D.cpp index 673e373dcf..569eaf994b 100644 --- a/tests/src/texture/simpleTexture3D.cpp +++ b/tests/src/texture/simpleTexture3D.cpp @@ -84,10 +84,10 @@ void runTest(int width,int height,int depth,texturechannelDesc; hipArray *arr; - HIPCHECK(hipMalloc3DArray(&arr, &channelDesc, make_hipExtent(width, height, depth), hipArrayCubemap)); + HIPCHECK(hipMalloc3DArray(&arr, &channelDesc, make_hipExtent(width, height, depth), hipArrayDefault)); hipMemcpy3DParms myparms = {0}; myparms.srcPos = make_hipPos(0,0,0); myparms.dstPos = make_hipPos(0,0,0); @@ -100,6 +100,7 @@ void runTest(int width,int height,int depth,textureaddressMode[0] = hipAddressModeWrap; tex->addressMode[1] = hipAddressModeWrap; + tex->addressMode[2] = hipAddressModeWrap; tex->filterMode = hipFilterModePoint; tex->normalized = false; From dd5f3fd282e6cd5900c7830b7ffd8b8b8781a5e1 Mon Sep 17 00:00:00 2001 From: Evgeny Mankov Date: Fri, 6 Mar 2020 18:17:05 +0300 Subject: [PATCH 078/154] [HIPIFY][doc] Update README.md: LLVM 10.0.0-rc3 is supported + Add -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON for LLVM 10.0.0 or newer + Supported versions update --- hipify-clang/README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/hipify-clang/README.md b/hipify-clang/README.md index a375c6ab65..88d7a72ccd 100644 --- a/hipify-clang/README.md +++ b/hipify-clang/README.md @@ -42,10 +42,10 @@ After applying all the matchers, the output HIP source is produced. `hipify-clang` requires: -1. [**LLVM+CLANG**](http://releases.llvm.org) of at least version [3.8.0](http://releases.llvm.org/download.html#3.8.0); the latest stable and recommended release: [**9.0.1**](http://releases.llvm.org/download.html#9.0.1), the latest release candidate: [10.0.0-rc2](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc2). +1. [**LLVM+CLANG**](http://releases.llvm.org) of at least version [3.8.0](http://releases.llvm.org/download.html#3.8.0); the latest stable and recommended release: [**9.0.1**](http://releases.llvm.org/download.html#9.0.1), the latest release candidate: [10.0.0-rc3](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc3). 2. [**CUDA**](https://developer.nvidia.com/cuda-downloads) of at least version [7.0](https://developer.nvidia.com/cuda-toolkit-70), the latest supported version is [**10.1 Update 2**](https://developer.nvidia.com/cuda-10.1-download-archive-base). -To use the latest CUDA version [10.2](https://developer.nvidia.com/cuda-downloads) please use the latest `LLVM` release candidate: [10.0.0-rc2](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc2). +To use the latest CUDA version [10.2](https://developer.nvidia.com/cuda-downloads) please use the latest `LLVM` release candidate: [10.0.0-rc3](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc3). | **LLVM release version** | **CUDA latest supported version** | **Windows** | **Linux** | |:----------------------------------------------------------:|:------------------------------------------------------------------------:|:-----------:|:---------:| @@ -67,7 +67,7 @@ To use the latest CUDA version [10.2](https://developer.nvidia.com/cuda-download | [8.0.1](http://releases.llvm.org/download.html#8.0.1) | [10.0](https://developer.nvidia.com/cuda-10.0-download-archive) | -
not working due to
the clang's bug [38811](https://bugs.llvm.org/show_bug.cgi?id=38811)
+
[patch](patches/patch_for_clang_8.0.1_bug_38811.zip)*
| + | | [9.0.0](http://releases.llvm.org/download.html#9.0.0) | [10.1](https://developer.nvidia.com/cuda-10.1-download-archive-base) | + | + | | [**9.0.1**](http://releases.llvm.org/download.html#9.0.1) | [**10.1**](https://developer.nvidia.com/cuda-10.1-download-archive-base) | +
**LATEST STABLE RELEASE** | +
**LATEST STABLE RELEASE** | -| [10.0.0-rc2](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc2) | [10.2](https://developer.nvidia.com/cuda-downloads) | + | + | +| [10.0.0-rc3](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0-rc3) | [10.2](https://developer.nvidia.com/cuda-downloads) | + | + | `*` Download the patch and unpack it into your `LLVM` distributive directory; a few header files will be overwritten; rebuilding of `LLVM` is not needed. @@ -158,7 +158,7 @@ Run `Visual Studio 16 2019`, open the generated `LLVM.sln`, build all, build pro **LLVM 10.0.0 or newer:** -1. download [`LLVM project`](https://github.com/llvm/llvm-project/archive/llvmorg-10.0.0-rc2.tar.gz) sources; +1. download [`LLVM project`](https://github.com/llvm/llvm-project/archive/llvmorg-10.0.0-rc3.tar.gz) sources; 2. build [`LLVM project`](http://llvm.org/docs/CMake.html): **Linux**: @@ -168,6 +168,7 @@ Run `Visual Studio 16 2019`, open the generated `LLVM.sln`, build all, build pro -DLLVM_SOURCE_DIR=../llvm-project \ -DLLVM_TARGETS_TO_BUILD="X86;NVPTX" \ -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON -DCMAKE_BUILD_TYPE=Release \ ../llvm-project/llvm make -j install @@ -181,6 +182,7 @@ Run `Visual Studio 16 2019`, open the generated `LLVM.sln`, build all, build pro -DLLVM_SOURCE_DIR=../llvm-project \ -DLLVM_TARGETS_TO_BUILD="NVPTX" \ -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON -DCMAKE_BUILD_TYPE=Release \ -Thost=x64 \ ../llvm-project/llvm @@ -247,7 +249,7 @@ On Linux the following configurations are tested: Ubuntu 14: LLVM 5.0.0 - 6.0.1, CUDA 7.0 - 9.0, cudnn-5.0.5 - cudnn-7.6.5.32 -Ubuntu 16-18: LLVM 8.0.0 - 10.0.0-rc2, CUDA 8.0 - 10.2, cudnn-5.1.10 - cudnn-7.6.5.32 +Ubuntu 16-18: LLVM 8.0.0 - 10.0.0-rc3, CUDA 8.0 - 10.2, cudnn-5.1.10 - cudnn-7.6.5.32 Minimum build system requirements for the above configurations: @@ -402,8 +404,8 @@ Testing Time: 3.07s | 7.0.0 - 7.1.0 | 9.2 | 7.6.5.32 | 2017.15.9.11 | 3.13.3 | 3.7.3 | | 8.0.0 - 8.0.1 | 10.0 | 7.6.5.32 | 2017.15.9.15 | 3.14.2 | 3.7.4 | | 9.0.0 - 9.0.1 | 10.1 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.0 | -| 10.0.0-rc1,rc2 | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.1 | -| 11.0.0git | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.1 | +| 10.0.0-rc1-rc3 | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.4 | 3.8.1 | +| 11.0.0git | 10.2 | 7.6.5.32 | 2017.15.9.20, 2019.16.4.5 | 3.16.5 | 3.8.2 | *Building with testing support on `Windows 10` by `Visual Studio 16 2019`:* @@ -429,7 +431,7 @@ cmake -- - CMake module path: F:/LLVM/9.0.1/dist/lib/cmake/llvm -- - Include path : F:/LLVM/9.0.1/dist/include -- - Binary path : F:/LLVM/9.0.1/dist/bin --- Found PythonInterp: C:/Program Files/Python38/python.exe (found suitable version "3.8.1", minimum required is "3.6") +-- Found PythonInterp: C:/Program Files/Python38/python.exe (found suitable version "3.8.2", minimum required is "3.6") -- Found lit: C:/Program Files/Python38/Scripts/lit.exe -- Found FileCheck: F:/LLVM/9.0.1/dist/bin/FileCheck.exe -- Found CUDA: C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1 (found version "10.1") From 90feaf046f740d84d64387f50e3d6813c9382cf5 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 6 Mar 2020 14:06:21 -0500 Subject: [PATCH 079/154] Add missing coversion cases. Handle converting signed int cases of hipResourceViewFormat to number of channels. Change-Id: Ica8ae6f644edfaa0d4803d0b8e90e320479118e2 --- vdi/hip_conversions.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index abaa4d1949..6809765eb0 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -349,6 +349,7 @@ int getNumChannels(const hipResourceViewFormat hipFormat) { case hipResViewFormatUnsignedShort1: case hipResViewFormatSignedShort1: case hipResViewFormatUnsignedInt1: + case hipResViewFormatSignedInt1: case hipResViewFormatHalf1: case hipResViewFormatFloat1: return 1; @@ -357,6 +358,7 @@ int getNumChannels(const hipResourceViewFormat hipFormat) { case hipResViewFormatUnsignedShort2: case hipResViewFormatSignedShort2: case hipResViewFormatUnsignedInt2: + case hipResViewFormatSignedInt2: case hipResViewFormatHalf2: case hipResViewFormatFloat2: return 2; @@ -365,6 +367,7 @@ int getNumChannels(const hipResourceViewFormat hipFormat) { case hipResViewFormatUnsignedShort4: case hipResViewFormatSignedShort4: case hipResViewFormatUnsignedInt4: + case hipResViewFormatSignedInt4: case hipResViewFormatHalf4: case hipResViewFormatFloat4: return 4; From dbb8f96a8e154c0eafaef2fa08dfc49247139a06 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 6 Mar 2020 14:10:56 -0500 Subject: [PATCH 080/154] Fix dangling pointer after hipUnbindTexture() call Change-Id: Ic4b476c62ebfae31e94dd139b20b6aaaa52bb866 --- vdi/hip_texture.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index bab61340ca..db405810a6 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -597,10 +597,13 @@ hipError_t hipUnbindTexture(const textureReference* texref) { HIP_INIT_API(hipUnbindTexture, texref); if (texref == nullptr) { - return hipErrorInvalidValue; + HIP_RETURN(hipErrorInvalidValue); } - HIP_RETURN(ihipDestroyTextureObject(texref->textureObject)); + const hipTextureObject_t textureObject = texref->textureObject; + const_cast(texref)->textureObject = nullptr; + + HIP_RETURN(ihipDestroyTextureObject(textureObject)); } hipError_t hipBindTexture(size_t* offset, From 2e9d1773041384cb6657a53be97463be9dc9c33c Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 6 Mar 2020 14:33:52 -0500 Subject: [PATCH 081/154] Add hipMemcpy3DParams -> HIP_MEMCPY3D conversion Change-Id: I66dd1ae722b8c0d7ddb2562c958b14854e6b86bc --- vdi/hip_conversions.hpp | 62 ++++++++++++++++++++++++++++++++++++++++- vdi/hip_memory.cpp | 59 ++------------------------------------- 2 files changed, 64 insertions(+), 57 deletions(-) diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index 6809765eb0..ff88ae4828 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -618,7 +618,7 @@ std::pair getMemoryType(const hipMemcpyKind kind) } inline -_HIP_MEMCPY3D getMemcpy3DParms(const hip_Memcpy2D& desc2D) { +_HIP_MEMCPY3D getDrvMemcpy3DDesc(const hip_Memcpy2D& desc2D) { _HIP_MEMCPY3D desc3D = {}; desc3D.srcXInBytes = desc2D.srcXInBytes; @@ -649,4 +649,64 @@ _HIP_MEMCPY3D getMemcpy3DParms(const hip_Memcpy2D& desc2D) { return desc3D; } + +inline +_HIP_MEMCPY3D getDrvMemcpy3DDesc(const hipMemcpy3DParms& desc) { + _HIP_MEMCPY3D descDrv = {}; + + descDrv.WidthInBytes = desc.extent.width; + descDrv.Height = desc.extent.height; + descDrv.Depth = desc.extent.depth; + + descDrv.srcXInBytes = desc.srcPos.x; + descDrv.srcY = desc.srcPos.y; + descDrv.srcZ = desc.srcPos.z; + descDrv.srcLOD = 0; + + descDrv.dstXInBytes = desc.dstPos.x; + descDrv.dstY = desc.dstPos.y; + descDrv.dstZ = desc.dstPos.z; + descDrv.dstLOD = 0; + + if (desc.srcArray != nullptr) { + descDrv.srcMemoryType = hipMemoryTypeArray; + descDrv.srcArray = desc.srcArray; + // When reffering to array memory, hipPos::x is in elements. + descDrv.srcXInBytes *= getElementSize(desc.srcArray->Format); + } + + if (desc.srcPtr.ptr != nullptr) { + descDrv.srcMemoryType = std::get<0>(hip::getMemoryType(desc.kind)); + descDrv.srcHost = desc.srcPtr.ptr; + descDrv.srcDevice = desc.srcPtr.ptr; + descDrv.srcPitch = desc.srcPtr.pitch; + descDrv.srcHeight = desc.srcPtr.ysize; + } + + if (desc.dstArray != nullptr) { + descDrv.dstMemoryType = hipMemoryTypeArray; + descDrv.dstArray = desc.dstArray; + // When reffering to array memory, hipPos::x is in elements. + descDrv.dstXInBytes *= getElementSize(desc.dstArray->Format); + } + + if (desc.dstPtr.ptr != nullptr) { + descDrv.dstMemoryType = std::get<1>(getMemoryType(desc.kind)); + descDrv.dstHost = desc.dstPtr.ptr; + descDrv.dstDevice = desc.dstPtr.ptr; + descDrv.dstPitch = desc.dstPtr.pitch; + descDrv.dstHeight = desc.dstPtr.ysize; + } + + // If a HIP array is participating in the copy, the extent is defined in terms of that array's elements. + if ((desc.srcArray != nullptr) && (desc.dstArray == nullptr)) { + descDrv.WidthInBytes *= getElementSize(desc.srcArray->Format); + } else if ((desc.srcArray == nullptr) && (desc.dstArray != nullptr)) { + descDrv.WidthInBytes *= getElementSize(desc.dstArray->Format); + } else if ((desc.srcArray != nullptr) && (desc.dstArray != nullptr)) { + descDrv.WidthInBytes *= getElementSize(desc.dstArray->Format); + } + + return descDrv; +} }; diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index cc3908ca64..84f8fa9def 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -1387,7 +1387,7 @@ hipError_t ihipMemcpyParam3D(const _HIP_MEMCPY3D* pCopy, hipError_t ihipMemcpyParam2D(const hip_Memcpy2D* pCopy, hipStream_t stream, bool isAsync = false) { - _HIP_MEMCPY3D desc = hip::getMemcpy3DParms(*pCopy); + _HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*pCopy); return ihipMemcpyParam3D(&desc, stream, isAsync); } @@ -1558,62 +1558,9 @@ hipError_t ihipMemcpy3D(const hipMemcpy3DParms* p, return hipErrorInvalidValue; } - _HIP_MEMCPY3D pCopy = {}; + const _HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*p); - pCopy.WidthInBytes = p->extent.width; - pCopy.Height = p->extent.height; - pCopy.Depth = p->extent.depth; - - pCopy.srcXInBytes = p->srcPos.x; - pCopy.srcY = p->srcPos.y; - pCopy.srcZ = p->srcPos.z; - pCopy.srcLOD = 0; - - pCopy.dstXInBytes = p->dstPos.x; - pCopy.dstY = p->dstPos.y; - pCopy.dstZ = p->dstPos.z; - pCopy.dstLOD = 0; - - if (p->srcArray != nullptr) { - pCopy.srcMemoryType = hipMemoryTypeArray; - pCopy.srcArray = p->srcArray; - // When reffering to array memory, hipPos::x is in elements. - pCopy.srcXInBytes *= hip::getElementSize(p->srcArray->Format); - } - - if (p->srcPtr.ptr != nullptr) { - pCopy.srcMemoryType = std::get<0>(hip::getMemoryType(p->kind)); - pCopy.srcHost = p->srcPtr.ptr; - pCopy.srcDevice = p->srcPtr.ptr; - pCopy.srcPitch = p->srcPtr.pitch; - pCopy.srcHeight = p->srcPtr.ysize; - } - - if (p->dstArray != nullptr) { - pCopy.dstMemoryType = hipMemoryTypeArray; - pCopy.dstArray = p->dstArray; - // When reffering to array memory, hipPos::x is in elements. - pCopy.dstXInBytes *= hip::getElementSize(p->dstArray->Format); - } - - if (p->dstPtr.ptr != nullptr) { - pCopy.dstMemoryType = std::get<1>(hip::getMemoryType(p->kind)); - pCopy.dstHost = p->dstPtr.ptr; - pCopy.dstDevice = p->dstPtr.ptr; - pCopy.dstPitch = p->dstPtr.pitch; - pCopy.dstHeight = p->dstPtr.ysize; - } - - // If a HIP array is participating in the copy, the extent is defined in terms of that array's elements. - if ((p->srcArray != nullptr) && (p->dstArray == nullptr)) { - pCopy.WidthInBytes *= hip::getElementSize(p->srcArray->Format); - } else if ((p->srcArray == nullptr) && (p->dstArray != nullptr)) { - pCopy.WidthInBytes *= hip::getElementSize(p->dstArray->Format); - } else if ((p->srcArray != nullptr) && (p->dstArray != nullptr)) { - pCopy.WidthInBytes *= hip::getElementSize(p->dstArray->Format); - } - - return ihipMemcpyParam3D(&pCopy, stream, isAsync); + return ihipMemcpyParam3D(&desc, stream, isAsync); } hipError_t hipMemcpy3D(const hipMemcpy3DParms* p) { From 38004b6ef0cc5a4fa3848ed4a0b6a9d420ad3253 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Mon, 9 Mar 2020 11:45:31 -0400 Subject: [PATCH 082/154] Fix Windows build. extern "C" on Windows implies nothrow. We shouldn't be throwing exceptions either way. Change-Id: If0ed1f7ec194bf7f65b7cea1a5c250e768a8f190 --- vdi/hip_rtc.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vdi/hip_rtc.cpp b/vdi/hip_rtc.cpp index 8bac9ae361..1f9a6e1636 100644 --- a/vdi/hip_rtc.cpp +++ b/vdi/hip_rtc.cpp @@ -193,8 +193,12 @@ const char* hiprtcGetErrorString(hiprtcResult x) { case HIPRTC_ERROR_INTERNAL_ERROR: return "HIPRTC_ERROR_INTERNAL_ERROR"; default: - throw std::logic_error{"Invalid HIPRTC result."}; + return nullptr; }; + + ShouldNotReachHere(); + + return nullptr; } hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, const char* src, const char* name, From 08c6d941ca13539bd6f1bf7a36c1445dbda9dd8a Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Fri, 28 Feb 2020 10:34:56 -0800 Subject: [PATCH 083/154] Fix HIPRTC headers to export C style symbols Change-Id: I3e0d2b19ace4a9096e3e46bd22f420483da51a8a --- vdi/hip_hcc.map.in | 14 ++--------- vdi/hip_rtc.cpp | 55 +++++++++++++++++++++++++---------------- vdi/hiprtc_internal.hpp | 14 ++++++++++- 3 files changed, 49 insertions(+), 34 deletions(-) diff --git a/vdi/hip_hcc.map.in b/vdi/hip_hcc.map.in index 043ea856ff..03d48fe9e7 100644 --- a/vdi/hip_hcc.map.in +++ b/vdi/hip_hcc.map.in @@ -181,6 +181,8 @@ global: hiprtcGetCode; hiprtcGetCodeSize; hiprtcGetErrorString; + hiprtcAddNameExpression; + hiprtcVersion; extern "C++" { hip_impl::hipLaunchKernelGGLImpl*; hip_impl::demangle*; @@ -221,18 +223,6 @@ global: hipInitActivityCallback*; hipEnableActivityCallback*; hipGetCmdName*; - hiprtcAddNameExpression*; - hiprtcCompileProgram*; - hiprtcCreateProgram*; - hiprtcDestroyProgram*; - hiprtcGetLoweredName*; - hiprtcGetProgramLog*; - hiprtcGetProgramLogSize*; - hiprtcGetCode*; - hiprtcGetCodeSize*; - hiprtcGetErrorString*; - hiprtcVersion*; - hiprtcGetTypeName*; }; local: *; diff --git a/vdi/hip_rtc.cpp b/vdi/hip_rtc.cpp index 1f9a6e1636..4ec48f3af3 100644 --- a/vdi/hip_rtc.cpp +++ b/vdi/hip_rtc.cpp @@ -23,10 +23,6 @@ #include #include "platform/program.hpp" -extern "C" char * __cxa_demangle(const char *mangled_name, char *output_buffer, - size_t *length, int *status); - - namespace hiprtc { thread_local hiprtcResult g_lastRtcError = HIPRTC_SUCCESS; } @@ -105,30 +101,34 @@ uint32_t ProgramState::addNameExpression(const char* name_expression) { return nameExpresssion_.size(); } -namespace hip_impl { - char* demangle(const char* loweredName) { -#ifdef ATI_OS_LINUX if (!loweredName) { return nullptr; } - +#if HIPRTC_USE_CXXABI || __linux__ int status = 0; - char* demangledName = __cxa_demangle(loweredName, nullptr, nullptr, &status); + char* demangledName = abi::__cxa_demangle(loweredName, nullptr, nullptr, &status); if (status != 0) { return nullptr; } +#elif defined(_WIN32) + char* demangledName = (char*)malloc(UNDECORATED_SIZE); - return demangledName; + if (!UnDecorateSymbolName(loweredName, demangledName, + UNDECORATED_SIZE/ sizeof(*demangledName), UNDNAME_COMPLETE)) + { + free(demangledName); + return nullptr; + } #else - return nullptr; -#endif +#error "Only Linux and Windows are supported" +#endif // HIPRTC_USE_CXXABI || __linux__ + return demangledName; } -} // hip_impl static std::string handleMangledName(std::string name) { std::string loweredName; - char* demangled = hip_impl::demangle(name.c_str()); + char* demangled = demangle(name.c_str()); loweredName.assign(demangled == nullptr ? std::string() : demangled); free(demangled); @@ -165,7 +165,6 @@ static std::string handleMangledName(std::string name) { return loweredName; } - const char* hiprtcGetErrorString(hiprtcResult x) { switch (x) { case HIPRTC_SUCCESS: @@ -232,7 +231,6 @@ hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, const char* src, const cha HIPRTC_RETURN(HIPRTC_SUCCESS); } - hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, int numOptions, const char** options) { // FIXME[skudchad] Add headers to amd::Program::build and device::Program::build, @@ -244,6 +242,8 @@ hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, int numOptions, const char std::ostringstream ostrstr; std::vector oarr(&options[0], &options[numOptions]); std::copy(oarr.begin(), oarr.end(), std::ostream_iterator(ostrstr, " ")); + ostrstr.str().append(" -DHIP_VERSION_MAJOR=").append(std::to_string(HIP_VERSION_MAJOR)); + ostrstr.str().append(" -DHIP_VERSION_MINOR=").append(std::to_string(HIP_VERSION_MINOR)); std::vector devices{hip::getCurrentDevice()->devices()[0]}; if (CL_SUCCESS != program->build(devices, ostrstr.str().c_str(), nullptr, nullptr)) { @@ -305,7 +305,7 @@ hiprtcResult hiprtcGetLoweredName(hiprtcProgram prog, const char* name_expressio *loweredName = it->second.second.c_str(); - HIPRTC_RETURN(HIPRTC_SUCCESS); + HIPRTC_RETURN(HIPRTC_SUCCESS); } hiprtcResult hiprtcDestroyProgram(hiprtcProgram* prog) { @@ -320,7 +320,7 @@ hiprtcResult hiprtcDestroyProgram(hiprtcProgram* prog) { program->release(); - HIPRTC_RETURN(HIPRTC_SUCCESS); + HIPRTC_RETURN(HIPRTC_SUCCESS); } hiprtcResult hiprtcGetCode(hiprtcProgram prog, char* binaryMem) { @@ -345,7 +345,7 @@ hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, size_t* binarySizeRet) { *binarySizeRet = program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0])->binary().second; - HIPRTC_RETURN(HIPRTC_SUCCESS); + HIPRTC_RETURN(HIPRTC_SUCCESS); } hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* dst) { @@ -360,7 +360,7 @@ hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* dst) { log.copy(dst, log.size()); dst[log.size()] = '\0'; - HIPRTC_RETURN(HIPRTC_SUCCESS); + HIPRTC_RETURN(HIPRTC_SUCCESS); } hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog, size_t* logSizeRet) { @@ -375,5 +375,18 @@ hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog, size_t* logSizeRet) { *logSizeRet = log.size() + 1; - HIPRTC_RETURN(HIPRTC_SUCCESS); + HIPRTC_RETURN(HIPRTC_SUCCESS); +} + +hiprtcResult hiprtcVersion(int* major, int* minor) { + HIPRTC_INIT_API(major, minor); + + if (major == nullptr || minor == nullptr) { + HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT); + } + + *major = 9; + *minor = 0; + + HIPRTC_RETURN(HIPRTC_SUCCESS); } diff --git a/vdi/hiprtc_internal.hpp b/vdi/hiprtc_internal.hpp index 2b9bdf026a..e95a28c08f 100644 --- a/vdi/hiprtc_internal.hpp +++ b/vdi/hiprtc_internal.hpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2019-present Advanced Micro Devices, Inc. +/* Copyright (c) 2015-present Advanced Micro Devices, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -23,6 +23,18 @@ #include "hip_internal.hpp" +#if HIPRTC_USE_CXXABI || __linux__ +#include +#include + +#elif defined(_WIN32) +#include +#include + +#define UNDECORATED_SIZE 4096 + +#endif // HIPRTC_USE_CXXABI || __linux__ + // This macro should be called at the beginning of every HIP RTC API. #define HIPRTC_INIT_API(...) \ ClPrint(amd::LOG_INFO, amd::LOG_API, "[%zx] %s ( %s )", std::this_thread::get_id(), __func__, ToString( __VA_ARGS__ ).c_str()); \ From c540a55d21a4d283b7cccd54d5676937837571d2 Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Mon, 9 Mar 2020 11:28:55 -0700 Subject: [PATCH 084/154] Change HIPRTC Version to 9.0 Change-Id: I7e88df61248f0fe6e33c2315805f9e49a493cf29 --- vdi/hip_rtc.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vdi/hip_rtc.cpp b/vdi/hip_rtc.cpp index 4ec48f3af3..3ccab6f44c 100644 --- a/vdi/hip_rtc.cpp +++ b/vdi/hip_rtc.cpp @@ -242,8 +242,9 @@ hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, int numOptions, const char std::ostringstream ostrstr; std::vector oarr(&options[0], &options[numOptions]); std::copy(oarr.begin(), oarr.end(), std::ostream_iterator(ostrstr, " ")); - ostrstr.str().append(" -DHIP_VERSION_MAJOR=").append(std::to_string(HIP_VERSION_MAJOR)); - ostrstr.str().append(" -DHIP_VERSION_MINOR=").append(std::to_string(HIP_VERSION_MINOR)); + + ostrstr.str().append(" -DHIP_VERSION_MAJOR=9"); + ostrstr.str().append(" -DHIP_VERSION_MINOR=0"); std::vector devices{hip::getCurrentDevice()->devices()[0]}; if (CL_SUCCESS != program->build(devices, ostrstr.str().c_str(), nullptr, nullptr)) { From c7d7640d87e29b19e536197e6ae94429ba503c6d Mon Sep 17 00:00:00 2001 From: Sameer Sahasrabuddhe Date: Wed, 4 Mar 2020 14:54:59 +0530 Subject: [PATCH 085/154] enable hostcall tests Change-Id: Ic6efe71c868defc53be214acd7e0907109ec1410 --- tests/src/hostcall/hipHostcallFuncCall.cpp | 2 +- tests/src/hostcall/hipHostcallPrintThings.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/src/hostcall/hipHostcallFuncCall.cpp b/tests/src/hostcall/hipHostcallFuncCall.cpp index 4b73352bb7..d849e69788 100644 --- a/tests/src/hostcall/hipHostcallFuncCall.cpp +++ b/tests/src/hostcall/hipHostcallFuncCall.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s EXCLUDE_HIP_PLATFORM all + * BUILD: %t %s EXCLUDE_HIP_PLATFORM NVCC EXCLUDE_HIP_RUNTIME HCC EXCLUDE_HIP_COMPILER hcc * HIT_END */ diff --git a/tests/src/hostcall/hipHostcallPrintThings.cpp b/tests/src/hostcall/hipHostcallPrintThings.cpp index 742798c0b9..8e3ebdae78 100644 --- a/tests/src/hostcall/hipHostcallPrintThings.cpp +++ b/tests/src/hostcall/hipHostcallPrintThings.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s EXCLUDE_HIP_PLATFORM all + * BUILD: %t %s EXCLUDE_HIP_PLATFORM NVCC EXCLUDE_HIP_RUNTIME HCC EXCLUDE_HIP_COMPILER hcc * HIT_END */ From 47cc94b1b4259f7400767af6beed2ee1ff9521e8 Mon Sep 17 00:00:00 2001 From: jujiang Date: Mon, 9 Mar 2020 12:46:00 -0400 Subject: [PATCH 086/154] SWDEV-225337-Fix test failure in hipNormalizedFloatValueTex kernel Change-Id: I3a4c3b58578703993640a8d28242ec1a0ed5ff60 --- tests/src/texture/hipNormalizedFloatValueTex.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/src/texture/hipNormalizedFloatValueTex.cpp b/tests/src/texture/hipNormalizedFloatValueTex.cpp index f8526eb145..04660ad70b 100644 --- a/tests/src/texture/hipNormalizedFloatValueTex.cpp +++ b/tests/src/texture/hipNormalizedFloatValueTex.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ @@ -55,14 +55,14 @@ __global__ void normalizedValTextureTest(unsigned int numElements, float* pDst) unsigned int elementID = hipThreadIdx_x; if(elementID >= numElements) return; - float coord =(float) elementID/(numElements-1); + float coord =(float) elementID/numElements; pDst[elementID] = tex1D(textureNormalizedVal_1D, coord); } template bool textureTest(enum hipArray_Format texFormat) { - T hData[] = {65, 66, 67, 68, 69, 70, 71, 72,73,74}; + T hData[] = {65, 66, 67, 68, 69, 70, 71, 72, 73, 74}; T *dData = NULL; HIPCHECK(hipMalloc((void **) &dData, sizeof(T)*SIZE)); HIPCHECK(hipMemcpyHtoD((hipDeviceptr_t)dData, hData, sizeof(T)*SIZE)); @@ -73,12 +73,8 @@ bool textureTest(enum hipArray_Format texFormat) HIPCHECK(hipTexRefSetFlags(texRef, HIP_TRSF_NORMALIZED_COORDINATES)); HIPCHECK(hipTexRefSetFormat(texRef, texFormat, 1)); - HIP_ARRAY_DESCRIPTOR desc; - desc.Width = SIZE; - desc.Height = 1; - desc.Format = texFormat; - desc.NumChannels = 1; - HIPCHECK(hipTexRefSetAddress2D(texRef, &desc, (hipDeviceptr_t)dData, sizeof(T)*SIZE)); + size_t offSet = 0; + HIPCHECK(hipTexRefSetAddress(&offSet, texRef, (hipDeviceptr_t)dData, sizeof(T)*SIZE)); bool testResult = true; float *dOutputData = NULL; @@ -101,7 +97,6 @@ bool textureTest(enum hipArray_Format texFormat) } hipFree(dData); hipFree(dOutputData); - hipUnbindTexture(textureNormalizedVal_1D); delete [] hOutputData; return testResult; } From ecd7c99b49160866bba4d17501de52c3ac78728b Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Mon, 9 Mar 2020 14:05:16 -0400 Subject: [PATCH 087/154] Add hipDrvMemcpy3D. This is the equivalent of cuMemcpy3D. Change-Id: Ib2e06dbd6f5093c931cdfd36c87617f32acffc2d --- bin/hipify-perl | 6 ++++ include/hip/hcc_detail/driver_types.h | 43 +++++++++++++----------- include/hip/hcc_detail/hip_runtime_api.h | 25 ++++++++++++++ vdi/hip_conversions.hpp | 38 +++------------------ vdi/hip_hcc.def.in | 2 ++ vdi/hip_hcc.map.in | 2 ++ vdi/hip_memory.cpp | 10 +++--- 7 files changed, 67 insertions(+), 59 deletions(-) diff --git a/bin/hipify-perl b/bin/hipify-perl index f6de5abae4..62cd8436f8 100755 --- a/bin/hipify-perl +++ b/bin/hipify-perl @@ -223,6 +223,10 @@ sub simpleSubstitutions { $ft{'memory'} += s/\bcuMemcpy2DAsync\b/hipMemcpyParam2DAsync/g; $ft{'memory'} += s/\bcuMemcpy2DAsync_v2\b/hipMemcpyParam2DAsync/g; $ft{'memory'} += s/\bcuMemcpy2D_v2\b/hipMemcpyParam2D/g; + $ft{'memory'} += s/\bcuMemcpy3D\b/hipDrvMemcpy3D/g; + $ft{'memory'} += s/\bcuMemcpy3DAsync\b/hipDrvMemcpy3DAsync/g; + $ft{'memory'} += s/\bcuMemcpy3D_v2\b/hipDrvMemcpy3D/g; + $ft{'memory'} += s/\bcuMemcpy3DAsync_v2\b/hipDrvMemcpy3DAsync/g; $ft{'memory'} += s/\bcuMemcpyAtoH\b/hipMemcpyAtoH/g; $ft{'memory'} += s/\bcuMemcpyAtoH_v2\b/hipMemcpyAtoH/g; $ft{'memory'} += s/\bcuMemcpyDtoD\b/hipMemcpyDtoD/g; @@ -938,6 +942,8 @@ sub simpleSubstitutions { $ft{'type'} += s/\bCUDA_ARRAY_DESCRIPTOR_st\b/HIP_ARRAY_DESCRIPTOR/g; $ft{'type'} += s/\bCUDA_MEMCPY2D\b/hip_Memcpy2D/g; $ft{'type'} += s/\bCUDA_MEMCPY2D_st\b/hip_Memcpy2D/g; + $ft{'type'} += s/\bCUDA_MEMCPY3D\b/HIP_MEMCPY3D/g; + $ft{'type'} += s/\bCUDA_MEMCPY3D_st\b/HIP_MEMCPY3D/g; $ft{'type'} += s/\bCUaddress_mode\b/hipTextureAddressMode/g; $ft{'type'} += s/\bCUaddress_mode_enum\b/hipTextureAddressMode/g; $ft{'type'} += s/\bCUarray\b/hipArray */g; diff --git a/include/hip/hcc_detail/driver_types.h b/include/hip/hcc_detail/driver_types.h index 1941f44617..d24f097f84 100644 --- a/include/hip/hcc_detail/driver_types.h +++ b/include/hip/hcc_detail/driver_types.h @@ -263,26 +263,29 @@ typedef struct hipMemcpy3DParms { } hipMemcpy3DParms; typedef struct HIP_MEMCPY3D { - size_t Depth; - size_t Height; - size_t WidthInBytes; - hipDeviceptr_t dstDevice; - size_t dstHeight; - void* dstHost; - size_t dstLOD; - hipMemoryType dstMemoryType; - size_t dstPitch; - size_t dstXInBytes; - size_t dstY; - size_t dstZ; - void* reserved0; - void* reserved1; - hipDeviceptr_t srcDevice; - size_t srcHeight; - const void* srcHost; - size_t srcLOD; - hipMemoryType srcMemoryType; - size_t srcPitch; + unsigned int srcXInBytes; + unsigned int srcY; + unsigned int srcZ; + unsigned int srcLOD; + hipMemoryType srcMemoryType; + const void* srcHost; + hipDeviceptr_t srcDevice; + hipArray_t srcArray; + unsigned int srcPitch; + unsigned int srcHeight; + unsigned int dstXInBytes; + unsigned int dstY; + unsigned int dstZ; + unsigned int dstLOD; + hipMemoryType dstMemoryType; + void* dstHost; + hipDeviceptr_t dstDevice; + hipArray_t dstArray; + unsigned int dstPitch; + unsigned int dstHeight; + unsigned int WidthInBytes; + unsigned int Height; + unsigned int Depth; } HIP_MEMCPY3D; static inline struct hipPitchedPtr make_hipPitchedPtr(void* d, size_t p, size_t xsz, diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 1f0b474863..2360333b1a 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -2165,6 +2165,31 @@ hipError_t hipMemcpy3D(const struct hipMemcpy3DParms* p); */ hipError_t hipMemcpy3DAsync(const struct hipMemcpy3DParms* p, hipStream_t stream __dparm(0)); +/** + * @brief Copies data between host and device. + * + * @param[in] pCopy 3D memory copy parameters + * @return #hipSuccess, #hipErrorInvalidValue, #hipErrorInvalidPitchValue, + * #hipErrorInvalidDevicePointer, #hipErrorInvalidMemcpyDirection + * + * @see hipMemcpy, hipMemcpy2DToArray, hipMemcpy2D, hipMemcpyFromArray, hipMemcpyToSymbol, + * hipMemcpyAsync + */ +hipError_t hipDrvMemcpy3D(const HIP_MEMCPY3D* pCopy); + +/** + * @brief Copies data between host and device asynchronously. + * + * @param[in] pCopy 3D memory copy parameters + * @param[in] stream Stream to use + * @return #hipSuccess, #hipErrorInvalidValue, #hipErrorInvalidPitchValue, + * #hipErrorInvalidDevicePointer, #hipErrorInvalidMemcpyDirection + * + * @see hipMemcpy, hipMemcpy2DToArray, hipMemcpy2D, hipMemcpyFromArray, hipMemcpyToSymbol, + * hipMemcpyAsync + */ +hipError_t hipDrvMemcpy3DAsync(const HIP_MEMCPY3D* pCopy, hipStream_t stream); + // doxygen end Memory /** * @} diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index ff88ae4828..c1ed166e16 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -25,36 +25,6 @@ THE SOFTWARE. #include #include -// HIP_MEMCPY3D is currently broken. -// TODO remove this struct once the headers will be fixed. -struct _HIP_MEMCPY3D { - unsigned int srcXInBytes; - unsigned int srcY; - unsigned int srcZ; - unsigned int srcLOD; - hipMemoryType srcMemoryType; - const void* srcHost; - hipDeviceptr_t srcDevice; - hipArray_t srcArray; - unsigned int srcPitch; - unsigned int srcHeight; - - unsigned int dstXInBytes; - unsigned int dstY; - unsigned int dstZ; - unsigned int dstLOD; - hipMemoryType dstMemoryType; - void* dstHost; - hipDeviceptr_t dstDevice; - hipArray_t dstArray; - unsigned int dstPitch; - unsigned int dstHeight; - - unsigned int WidthInBytes; - unsigned int Height; - unsigned int Depth; -}; - namespace hip { inline @@ -618,8 +588,8 @@ std::pair getMemoryType(const hipMemcpyKind kind) } inline -_HIP_MEMCPY3D getDrvMemcpy3DDesc(const hip_Memcpy2D& desc2D) { - _HIP_MEMCPY3D desc3D = {}; +HIP_MEMCPY3D getDrvMemcpy3DDesc(const hip_Memcpy2D& desc2D) { + HIP_MEMCPY3D desc3D = {}; desc3D.srcXInBytes = desc2D.srcXInBytes; desc3D.srcY = desc2D.srcY; @@ -651,8 +621,8 @@ _HIP_MEMCPY3D getDrvMemcpy3DDesc(const hip_Memcpy2D& desc2D) { } inline -_HIP_MEMCPY3D getDrvMemcpy3DDesc(const hipMemcpy3DParms& desc) { - _HIP_MEMCPY3D descDrv = {}; +HIP_MEMCPY3D getDrvMemcpy3DDesc(const hipMemcpy3DParms& desc) { + HIP_MEMCPY3D descDrv = {}; descDrv.WidthInBytes = desc.extent.width; descDrv.Height = desc.extent.height; diff --git a/vdi/hip_hcc.def.in b/vdi/hip_hcc.def.in index 54492eb48e..4ec2f698c0 100644 --- a/vdi/hip_hcc.def.in +++ b/vdi/hip_hcc.def.in @@ -89,6 +89,8 @@ hipMemcpy2DAsync hipMemcpy2DToArray hipMemcpy3D hipMemcpy3DAsync +hipDrvMemcpy3D +hipDrvMemcpy3DAsync hipMemcpyAsync hipMemcpyDtoD hipMemcpyDtoDAsync diff --git a/vdi/hip_hcc.map.in b/vdi/hip_hcc.map.in index 03d48fe9e7..b8ee45d647 100644 --- a/vdi/hip_hcc.map.in +++ b/vdi/hip_hcc.map.in @@ -90,6 +90,8 @@ global: hipMemcpy2DToArray; hipMemcpy3D; hipMemcpy3DAsync; + hipDrvMemcpy3D; + hipDrvMemcpy3DAsync; hipMemcpyAsync; hipMemcpyDtoD; hipMemcpyDtoDAsync; diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 84f8fa9def..55bc11512a 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -1332,7 +1332,7 @@ hipError_t ihipMemcpyAtoH(hipArray* srcArray, return hipSuccess; } -hipError_t ihipMemcpyParam3D(const _HIP_MEMCPY3D* pCopy, +hipError_t ihipMemcpyParam3D(const HIP_MEMCPY3D* pCopy, hipStream_t stream, bool isAsync = false) { // If {src/dst}MemoryType is hipMemoryTypeUnified, {src/dst}Device and {src/dst}Pitch specify the (unified virtual address space) @@ -1387,7 +1387,7 @@ hipError_t ihipMemcpyParam3D(const _HIP_MEMCPY3D* pCopy, hipError_t ihipMemcpyParam2D(const hip_Memcpy2D* pCopy, hipStream_t stream, bool isAsync = false) { - _HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*pCopy); + HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*pCopy); return ihipMemcpyParam3D(&desc, stream, isAsync); } @@ -1558,7 +1558,7 @@ hipError_t ihipMemcpy3D(const hipMemcpy3DParms* p, return hipErrorInvalidValue; } - const _HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*p); + const HIP_MEMCPY3D desc = hip::getDrvMemcpy3DDesc(*p); return ihipMemcpyParam3D(&desc, stream, isAsync); } @@ -1575,13 +1575,13 @@ hipError_t hipMemcpy3DAsync(const hipMemcpy3DParms* p, hipStream_t stream) { HIP_RETURN(ihipMemcpy3D(p, stream, true)); } -hipError_t hipDrvMemcpy3D(const _HIP_MEMCPY3D* pCopy) { +hipError_t hipDrvMemcpy3D(const HIP_MEMCPY3D* pCopy) { HIP_INIT_API(hipDrvMemcpy3D, pCopy); HIP_RETURN(ihipMemcpyParam3D(pCopy, nullptr)); } -hipError_t hipDrvMemcpy3DAsync(const _HIP_MEMCPY3D* pCopy, hipStream_t stream) { +hipError_t hipDrvMemcpy3DAsync(const HIP_MEMCPY3D* pCopy, hipStream_t stream) { HIP_INIT_API(hipDrvMemcpy3DAsync, pCopy, stream); HIP_RETURN(ihipMemcpyParam3D(pCopy, stream, true)); From aa04b964cf093d6db6075942335bc264563c28b6 Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Tue, 10 Mar 2020 00:02:10 -0700 Subject: [PATCH 088/154] Fix P4 linux make builds Change-Id: I6cc4e69a914389b53bc3b52535eb6faf24897a09 --- vdi/hip_rtc.cpp | 6 +++--- vdi/hiprtc_internal.hpp | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/vdi/hip_rtc.cpp b/vdi/hip_rtc.cpp index 3ccab6f44c..9897b98b7f 100644 --- a/vdi/hip_rtc.cpp +++ b/vdi/hip_rtc.cpp @@ -105,9 +105,9 @@ char* demangle(const char* loweredName) { if (!loweredName) { return nullptr; } -#if HIPRTC_USE_CXXABI || __linux__ +#if __linux__ int status = 0; - char* demangledName = abi::__cxa_demangle(loweredName, nullptr, nullptr, &status); + char* demangledName = DEMANGLE(loweredName, nullptr, nullptr, &status); if (status != 0) { return nullptr; } @@ -122,7 +122,7 @@ char* demangle(const char* loweredName) { } #else #error "Only Linux and Windows are supported" -#endif // HIPRTC_USE_CXXABI || __linux__ +#endif // __linux__ return demangledName; } diff --git a/vdi/hiprtc_internal.hpp b/vdi/hiprtc_internal.hpp index e95a28c08f..4e533c9716 100644 --- a/vdi/hiprtc_internal.hpp +++ b/vdi/hiprtc_internal.hpp @@ -23,17 +23,28 @@ #include "hip_internal.hpp" -#if HIPRTC_USE_CXXABI || __linux__ -#include +#if __linux__ #include +#if HIPRTC_USE_CXXABI +#include + +#define DEMANGLE abi::__cxa_demangle + +#else +extern "C" char * __cxa_demangle(const char *mangled_name, char *output_buffer, + size_t *length, int *status); + +#define DEMANGLE __cxa_demangle +#endif //HIPRTC_USE_CXXABI + #elif defined(_WIN32) #include #include #define UNDECORATED_SIZE 4096 -#endif // HIPRTC_USE_CXXABI || __linux__ +#endif // __linux__ // This macro should be called at the beginning of every HIP RTC API. #define HIPRTC_INIT_API(...) \ From 92af5e43755887379fd1f5e01dfff9a75d50d1aa Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Tue, 10 Mar 2020 13:01:28 -0400 Subject: [PATCH 089/154] Let hipcc not pass -mllvm option to HIP-Clang on Windows Currently there is a clang bug on Windows causing duplicate -mllvm options in clang -cc1. Tempoarily disable -mllvm options for HIP-Clang on Windows until the bug is fixed. Change-Id: I3a4393ba7745989398dc6c6001722837dad18704 --- bin/hipcc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/hipcc b/bin/hipcc index 33a07c39dc..83774a996b 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -726,10 +726,12 @@ if ($HIP_PLATFORM eq "clang") { $HIPCXXFLAGS .= " -O3"; $HIPLDFLAGS .= " -O3"; } - if ($optArg ne "-O0") { - $HIPCXXFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; - if ($needLDFLAGS and not $needCXXFLAGS) { - $HIPLDFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; + # Do not pass -mllvm on Windows since there is a clang bug causing duplicate -mllvm options in clang -cc1 on Windows. + # ToDo : remove restriction for Windows after clang bug is fixed. + if ($optArg ne "-O0" and not $isWindows) { + $HIPCXXFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; + if ($needLDFLAGS and not $needCXXFLAGS) { + $HIPLDFLAGS .= " -mllvm -amdgpu-early-inline-all=true -mllvm -amdgpu-function-calls=false"; } } $HIP_DEVLIB_FLAGS = " --hip-device-lib-path=$DEVICE_LIB_PATH"; From 604befb472ca1f8f6917ac218c3a631b4d8fb2e3 Mon Sep 17 00:00:00 2001 From: Jiabao Xie Date: Fri, 6 Mar 2020 16:44:43 -0500 Subject: [PATCH 090/154] Structs expanded to print struct information than address. Moved to hip_formatting.hpp. Reformatted to follow Google standards Change-Id: I08695058c11db51b9f3cbe1deb4af944ebf9e64d --- vdi/hip_formatting.hpp | 875 +++++++++++++++++++++++++++++++++++++++++ vdi/hip_internal.hpp | 12 +- 2 files changed, 876 insertions(+), 11 deletions(-) create mode 100644 vdi/hip_formatting.hpp diff --git a/vdi/hip_formatting.hpp b/vdi/hip_formatting.hpp new file mode 100644 index 0000000000..aab267d392 --- /dev/null +++ b/vdi/hip_formatting.hpp @@ -0,0 +1,875 @@ +/* Copyright (c) 2015-present Advanced Micro Devices, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. */ +#include "hip/hcc_detail/hiprtc.h" +#include "hip/hcc_detail/hip_runtime_prof.h" + +inline std::ostream& operator<<(std::ostream& os, const hipTextureFilterMode& s) { + switch (s) { + case hipFilterModePoint: + os << "hipFilterModePoint"; + break; + case hipFilterModeLinear: + os << "hipFilterModeLinear"; + break; + default: + os << "hipFilterModePoint"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipTextureReadMode& s) { + switch (s) { + case hipReadModeElementType: + os << "hipReadModeElementType"; + break; + case hipReadModeNormalizedFloat: + os << "hipReadModeNormalizedFloat"; + break; + default: + os << "hipReadModeElementType"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipTextureAddressMode& s) { + switch (s) { + case hipAddressModeWrap: + os << "hipAddressModeWrap"; + break; + case hipAddressModeClamp: + os << "hipAddressModeClamp"; + break; + case hipAddressModeMirror: + os << "hipAddressModeMirror"; + break; + case hipAddressModeBorder: + os << "hipAddressModeBorder"; + break; + default: + os << "hipAddressModeWrap"; + }; + return os; +} + + +inline std::ostream& operator<<(std::ostream& os, const hipMemcpyKind& s) { + switch (s) { + case hipMemcpyHostToHost: + os << "hipMemcpyHostToHost"; + break; + case hipMemcpyHostToDevice: + os << "hipMemcpyHostToDevice"; + break; + case hipMemcpyDeviceToHost: + os << "hipMemcpyDeviceToHost"; + break; + case hipMemcpyDeviceToDevice: + os << "hipMemcpyDeviceToDevice"; + break; + case hipMemcpyDefault: + os << "hipMemcpyDefault"; + break; + default: + os << "hipMemcpyDefault"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatKind& s) { + switch (s) { + case hipChannelFormatKindSigned: + os << "hipChannelFormatKindSigned"; + break; + case hipChannelFormatKindUnsigned: + os << "hipMemcpyHostToDevice"; + break; + case hipChannelFormatKindFloat: + os << "hipChannelFormatKindFloat"; + break; + case hipChannelFormatKindNone: + os << "hipChannelFormatKindNone"; + break; + default: + os << "hipChannelFormatKindNone"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipArray_Format& s) { + switch (s) { + case HIP_AD_FORMAT_UNSIGNED_INT8: + os << "HIP_AD_FORMAT_UNSIGNED_INT8"; + break; + case HIP_AD_FORMAT_UNSIGNED_INT16: + os << "HIP_AD_FORMAT_UNSIGNED_INT16"; + break; + case HIP_AD_FORMAT_UNSIGNED_INT32: + os << "HIP_AD_FORMAT_UNSIGNED_INT32"; + break; + case HIP_AD_FORMAT_SIGNED_INT8: + os << "HIP_AD_FORMAT_SIGNED_INT8"; + break; + case HIP_AD_FORMAT_SIGNED_INT16: + os << "HIP_AD_FORMAT_SIGNED_INT16"; + break; + case HIP_AD_FORMAT_SIGNED_INT32: + os << "HIP_AD_FORMAT_SIGNED_INT32"; + break; + case HIP_AD_FORMAT_HALF: + os << "HIP_AD_FORMAT_HALF"; + break; + case HIP_AD_FORMAT_FLOAT: + os << "HIP_AD_FORMAT_FLOAT"; + break; + default: + os << "HIP_AD_FORMAT_FLOAT"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipResourceViewFormat& s) { + switch (s) { + case hipResViewFormatNone: + os << "hipResViewFormatNone"; + break; + case hipResViewFormatUnsignedChar1: + os << "hipResViewFormatUnsignedChar1"; + break; + case hipResViewFormatUnsignedChar2: + os << "hipResViewFormatUnsignedChar2"; + break; + case hipResViewFormatUnsignedChar4: + os << "hipResViewFormatUnsignedChar4"; + break; + case hipResViewFormatSignedChar1: + os << "hipResViewFormatSignedChar1"; + break; + case hipResViewFormatSignedChar2: + os << "hipResViewFormatSignedChar2"; + break; + case hipResViewFormatSignedChar4: + os << "hipResViewFormatSignedChar4"; + break; + case hipResViewFormatUnsignedShort1: + os << "hipResViewFormatUnsignedShort1"; + break; + case hipResViewFormatUnsignedShort2: + os << "hipResViewFormatUnsignedShort2"; + break; + case hipResViewFormatUnsignedShort4: + os << "hipResViewFormatUnsignedShort4"; + break; + case hipResViewFormatSignedShort1: + os << "hipResViewFormatSignedShort1"; + break; + case hipResViewFormatSignedShort2: + os << "hipResViewFormatSignedShort2"; + break; + case hipResViewFormatSignedShort4: + os << "hipResViewFormatSignedShort4"; + break; + case hipResViewFormatUnsignedInt1: + os << "hipResViewFormatUnsignedInt1"; + break; + case hipResViewFormatUnsignedInt2: + os << "hipResViewFormatUnsignedInt2"; + break; + case hipResViewFormatUnsignedInt4: + os << "hipResViewFormatUnsignedInt4"; + break; + case hipResViewFormatSignedInt1: + os << "hipResViewFormatSignedInt1"; + break; + case hipResViewFormatSignedInt2: + os << "hipResViewFormatSignedInt2"; + break; + case hipResViewFormatSignedInt4: + os << "hipResViewFormatSignedInt4"; + break; + case hipResViewFormatHalf1: + os << "hipResViewFormatHalf1"; + break; + case hipResViewFormatHalf2: + os << "hipResViewFormatHalf2"; + break; + case hipResViewFormatHalf4: + os << "hipResViewFormatHalf4"; + break; + case hipResViewFormatFloat1: + os << "hipResViewFormatFloat1"; + break; + case hipResViewFormatFloat2: + os << "hipResViewFormatFloat2"; + break; + case hipResViewFormatFloat4: + os << "hipResViewFormatFloat4"; + break; + case hipResViewFormatUnsignedBlockCompressed1: + os << "hipResViewFormatUnsignedBlockCompressed1"; + break; + case hipResViewFormatUnsignedBlockCompressed2: + os << "hipResViewFormatUnsignedBlockCompressed2"; + break; + case hipResViewFormatUnsignedBlockCompressed3: + os << "hipResViewFormatUnsignedBlockCompressed3"; + break; + case hipResViewFormatUnsignedBlockCompressed4: + os << "hipResViewFormatUnsignedBlockCompressed4"; + break; + case hipResViewFormatSignedBlockCompressed4: + os << "hipResViewFormatSignedBlockCompressed4"; + break; + case hipResViewFormatUnsignedBlockCompressed5: + os << "hipResViewFormatUnsignedBlockCompressed5"; + break; + case hipResViewFormatSignedBlockCompressed5: + os << "hipResViewFormatSignedBlockCompressed5"; + break; + case hipResViewFormatUnsignedBlockCompressed6H: + os << "hipResViewFormatUnsignedBlockCompressed6H"; + break; + case hipResViewFormatSignedBlockCompressed6H: + os << "hipResViewFormatSignedBlockCompressed6H"; + break; + case hipResViewFormatUnsignedBlockCompressed7: + os << "hipResViewFormatUnsignedBlockCompressed7"; + break; + default: + os << "hipResViewFormatNone"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipFunction_attribute& s) { + switch (s) { + case HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK: + os << "HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK"; + break; + case HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES: + os << "HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES"; + break; + case HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES: + os << "HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES"; + break; + case HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES: + os << "HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES"; + break; + case HIP_FUNC_ATTRIBUTE_NUM_REGS: + os << "HIP_FUNC_ATTRIBUTE_NUM_REGS"; + break; + case HIP_FUNC_ATTRIBUTE_PTX_VERSION: + os << "HIP_FUNC_ATTRIBUTE_PTX_VERSION"; + break; + case HIP_FUNC_ATTRIBUTE_BINARY_VERSION: + os << "HIP_FUNC_ATTRIBUTE_BINARY_VERSION"; + break; + case HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA: + os << "HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA"; + break; + case HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES: + os << "HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES"; + break; + case HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT: + os << "HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT"; + break; + case HIP_FUNC_ATTRIBUTE_MAX: + os << "HIP_FUNC_ATTRIBUTE_MAX"; + break; + default: + os << "HIP_FUNC_ATTRIBUTE_MAX"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hiprtcResult& s) { + switch (s) { + case HIPRTC_SUCCESS: + os << "HIPRTC_SUCCESS"; + break; + case HIPRTC_ERROR_OUT_OF_MEMORY: + os << "HIPRTC_ERROR_OUT_OF_MEMORY"; + break; + case HIPRTC_ERROR_PROGRAM_CREATION_FAILURE: + os << "HIPRTC_ERROR_PROGRAM_CREATION_FAILURE"; + break; + case HIPRTC_ERROR_INVALID_INPUT: + os << "HIPRTC_ERROR_INVALID_INPUT"; + break; + case HIPRTC_ERROR_INVALID_PROGRAM: + os << "HIPRTC_ERROR_INVALID_PROGRAM"; + break; + case HIPRTC_ERROR_INVALID_OPTION: + os << "HIPRTC_ERROR_INVALID_OPTION"; + break; + case HIPRTC_ERROR_COMPILATION: + os << "HIPRTC_ERROR_COMPILATION"; + break; + case HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE: + os << "HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE"; + break; + case HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION: + os << "HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION"; + break; + case HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION: + os << "IPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION"; + break; + case HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID: + os << "HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID"; + break; + case HIPRTC_ERROR_INTERNAL_ERROR: + os << "HIPRTC_ERROR_INTERNAL_ERROR"; + break; + default: + os << "HIPRTC_ERROR_INTERNAL_ERROR"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const HipVdiOpId& s) { + switch (s) { + case kHipVdiOpIdDispatch: + os << "kHipVdiOpIdDispatch"; + break; + case kHipVdiOpIdCopy: + os << "kHipVdiOpIdCopy"; + break; + case kHipVdiOpIdBarrier: + os << "kHipVdiOpIdBarrier"; + break; + case kHipVdiOpIdNumber: + os << "kHipVdiOpIdNumber"; + break; + default: + os << "kHipVdiOpIdDispatch"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const HipVdiCommandKind& s) { + switch (s) { + case kHipVdiCommandKernel: + os << "kHipVdiCommandKernel"; + break; + case kHipVdiMemcpyDeviceToHost: + os << "kHipVdiMemcpyDeviceToHost"; + break; + case kHipHipVdiMemcpyHostToDevice: + os << "kHipHipVdiMemcpyHostToDevice"; + break; + case kHipVdiMemcpyDeviceToDevice: + os << "kHipVdiMemcpyDeviceToDevice"; + break; + case kHipVidMemcpyDeviceToHostRect: + os << "kHipVidMemcpyDeviceToHostRect"; + break; + case kHipVdiMemcpyHostToDeviceRect: + os << "kHipVdiMemcpyHostToDeviceRect"; + break; + case kHipVdiMemcpyDeviceToDeviceRect: + os << "kHipVdiMemcpyDeviceToDeviceRect"; + break; + case kHipVdiFillMemory: + os << "kHipVdiFillMemory"; + break; + default: + os << "kHipVdiCommandKernel"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipJitOption& s) { + switch (s) { + case hipJitOptionMaxRegisters: + os << "hipJitOptionMaxRegisters"; + break; + case hipJitOptionThreadsPerBlock: + os << "hipJitOptionThreadsPerBlock"; + break; + case hipJitOptionWallTime: + os << "hipJitOptionWallTime"; + break; + case hipJitOptionInfoLogBuffer: + os << "hipJitOptionInfoLogBuffer"; + break; + case hipJitOptionInfoLogBufferSizeBytes: + os << "hipJitOptionInfoLogBufferSizeBytes"; + break; + case hipJitOptionErrorLogBuffer: + os << "hipJitOptionErrorLogBuffer"; + break; + case hipJitOptionErrorLogBufferSizeBytes: + os << "hipJitOptionErrorLogBufferSizeBytes"; + break; + case hipJitOptionOptimizationLevel: + os << "hipJitOptionOptimizationLevel"; + break; + case hipJitOptionTargetFromContext: + os << "hipJitOptionTargetFromContext"; + break; + case hipJitOptionTarget: + os << "hipJitOptionTarget"; + break; + case hipJitOptionFallbackStrategy: + os << "hipJitOptionFallbackStrategy"; + break; + case hipJitOptionGenerateDebugInfo: + os << "hipJitOptionGenerateDebugInfo"; + break; + case hipJitOptionCacheMode: + os << "hipJitOptionCacheMode"; + break; + case hipJitOptionSm3xOpt: + os << "hipJitOptionSm3xOpt"; + break; + case hipJitOptionFastCompile: + os << "hipJitOptionFastCompile"; + break; + case hipJitOptionNumOptions: + os << "hipJitOptionNumOptions"; + break; + default: + os << "hipJitOptionMaxRegisters"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipFuncCache_t& s) { + switch (s) { + case hipFuncCachePreferNone: + os << "hipFuncCachePreferNone"; + break; + case hipFuncCachePreferShared: + os << "hipFuncCachePreferShared"; + break; + case hipFuncCachePreferL1: + os << "hipFuncCachePreferL1"; + break; + case hipFuncCachePreferEqual: + os << "hipFuncCachePreferEqual"; + break; + default: + os << "hipFuncCachePreferNone"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipSharedMemConfig& s) { + switch (s) { + case hipSharedMemBankSizeDefault: + os << "hipSharedMemBankSizeDefault"; + break; + case hipSharedMemBankSizeFourByte: + os << "hipSharedMemBankSizeFourByte"; + break; + case hipSharedMemBankSizeEightByte: + os << "hipSharedMemBankSizeEightByte"; + break; + default: + os << "hipSharedMemBankSizeDefault"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipDataType& s) { + switch (s) { + case HIP_R_16F: + os << "HIP_R_16F"; + break; + case HIP_R_32F: + os << "HIP_R_32F"; + break; + case HIP_R_64F: + os << "HIP_R_64F"; + break; + case HIP_C_16F: + os << "HIP_C_16F"; + break; + case HIP_C_32F: + os << "HIP_C_32F"; + break; + case HIP_C_64F: + os << "HIP_C_64F"; + break; + default: + os << "HIP_R_16F"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipLibraryPropertyType& s) { + switch (s) { + case HIP_LIBRARY_MAJOR_VERSION: + os << "HIP_LIBRARY_MAJOR_VERSION"; + break; + case HIP_LIBRARY_MINOR_VERSION: + os << "HIP_LIBRARY_MINOR_VERSION"; + break; + case HIP_LIBRARY_PATCH_LEVEL: + os << "HIP_LIBRARY_PATCH_LEVEL"; + break; + default: + os << "HIP_LIBRARY_MAJOR_VERSION"; + }; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hip_api_id_t& s) { + os << hip_api_name(s); + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hip_api_id_t* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipTextureDesc& s) { + os << '{' + << '{' + << s.addressMode[0] + << ',' + << s.addressMode[1] + << ',' + << s.addressMode[2] + << '}' + << ',' + << s.filterMode + << ',' + << s.readMode + << ',' + << s.sRGB + << ',' + << '{' + << s.borderColor[0] + << ',' + << s.borderColor[1] + << ',' + << s.borderColor[2] + << ',' + << s.borderColor[3] + << '}' + << ',' + << s.normalizedCoords + << ',' + << s.mipmapFilterMode + << ',' + << s.mipmapLevelBias + << ',' + << s.minMipmapLevelClamp + << ',' + << s.maxMipmapLevelClamp + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipTextureDesc* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + + +inline std::ostream& operator<<(std::ostream& os, const dim3& s) { + os << '{' + << s.x + << ',' + << s.y + << ',' + << s.z + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const dim3* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatDesc& s) { + os << '{' + << s.x + << ',' + << s.y + << ',' + << s.z + << ',' + << s.w + << ',' + << s.f + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatDesc* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipMipmappedArray& s) { + os << '{' + << s.data + << ',' + << s.desc + << ',' + << s.width + << ',' + << s.height + << ',' + << s.depth + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipMipmappedArray* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + + +inline std::ostream& operator<<(std::ostream& os, const hipResourceDesc& s) { + os << '{' + << s.resType + << ',' + << s.res.array.array + << ',' + << s.res.mipmap.mipmap + << ',' + << s.res.linear.devPtr + << ',' + << s.res.linear.desc + << ',' + << s.res.linear.sizeInBytes + << ',' + << s.res.pitch2D.devPtr + << ',' + << s.res.pitch2D.desc + << ',' + << s.res.pitch2D.width + << ',' + << s.res.pitch2D.height + << ',' + << s.res.pitch2D.pitchInBytes + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipResourceDesc* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + + +inline std::ostream& operator<<(std::ostream& os, const hipArray& s) { + os << '{' + << s.data + << ',' + << s.desc + << ',' + << s.type + << ',' + << s.width + << ',' + << s.height + << ',' + << s.depth + << ',' + << s.Format + << ',' + << s.NumChannels + << ',' + << s.isDrv + << ',' + << s.textureType + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipArray* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const textureReference& s) { + os << '{' + << s.normalized + << ',' + << s.readMode + << ',' + << s.filterMode + << ',' + << '{' + << s.addressMode[0] + << ',' + << s.addressMode[1] + << ',' + << s.addressMode[2] + << '}' + << ',' + << s.channelDesc + << ',' + << s.sRGB + << ',' + << s.maxAnisotropy + << ',' + << s.mipmapFilterMode + << ',' + << s.mipmapLevelBias + << ',' + << s.minMipmapLevelClamp + << ',' + << s.maxMipmapLevelClamp + << ',' + << s.textureObject + << ',' + << s.numChannels + << ',' + << s.format + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const textureReference* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + + +inline std::ostream& operator<<(std::ostream& os, const hipError_t& s) { + os << hipGetErrorName(s); + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipError_t* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipResourceViewDesc& s) { + os << '{' + << s.format + << ',' + << s.width + << ',' + << s.height + << ',' + << s.depth + << ',' + << s.firstMipmapLevel + << ',' + << s.lastMipmapLevel + << ',' + << s.firstLayer + << ',' + << s.lastLayer + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipResourceViewDesc* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY_DESCRIPTOR& s) { + os << '{' + << s.Width + << ',' + << s.Height + << ',' + << s.Format + << ',' + << s.NumChannels + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY_DESCRIPTOR* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY3D_DESCRIPTOR& s) { + os << '{' + << s.Width + << ',' + << s.Height + << ',' + << s.Depth + << ',' + << s.Format + << ',' + << s.NumChannels + << ',' + << s.Flags + << '}'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY3D_DESCRIPTOR* s) { + if (s) { + os << *s; + } else { + os << "nullptr"; + } + return os; +} \ No newline at end of file diff --git a/vdi/hip_internal.hpp b/vdi/hip_internal.hpp index 86012438fa..e11e316503 100644 --- a/vdi/hip_internal.hpp +++ b/vdi/hip_internal.hpp @@ -25,6 +25,7 @@ #include "hip_prof_api.h" #include "trace_helper.h" #include "utils/debug.hpp" +#include "hip_formatting.hpp" #include #include #include @@ -244,15 +245,4 @@ extern amd::Memory* getMemoryObject(const void* ptr, size_t& offset); extern bool CL_CALLBACK getSvarInfo(cl_program program, std::string var_name, void** var_addr, size_t* var_size); -inline std::ostream& operator<<(std::ostream& os, const dim3& s) { - os << '{'; - os << s.x; - os << ','; - os << s.y; - os << ','; - os << s.z; - os << '}'; - return os; -} - #endif // HIP_SRC_HIP_INTERNAL_H From 4ca9cda3722197777ec7b1e4cf371f8f77b5c5dc Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 10 Mar 2020 13:23:08 -0400 Subject: [PATCH 091/154] Fix typo in device __shfl_xor function Change-Id: I8bcdd53ced00c596a0af013a0c34e37aa67c93ae --- include/hip/hcc_detail/device_functions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/hip/hcc_detail/device_functions.h b/include/hip/hcc_detail/device_functions.h index 7096841da8..b96bbba07f 100644 --- a/include/hip/hcc_detail/device_functions.h +++ b/include/hip/hcc_detail/device_functions.h @@ -548,7 +548,7 @@ long __shfl_xor(long var, int lane_mask, int width = warpSize) return tmp1; #else static_assert(sizeof(long) == sizeof(int), ""); - return static_cast(__shfl_down(static_cast(var), lane_delta, width)); + return static_cast(__shfl_down(static_cast(var), lane_mask, width)); #endif } __device__ From e7abc7a553599fe1c40dff14426377b09f18a0a3 Mon Sep 17 00:00:00 2001 From: Jiabao Xie Date: Thu, 12 Mar 2020 14:13:01 -0400 Subject: [PATCH 092/154] Cleaned include statements and deleted unnecessary operator overloads Change-Id: I5eb322c05082e8f27584446af2f6a19243dc63c9 --- vdi/hip_formatting.hpp | 56 ++---------------------------------------- 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/vdi/hip_formatting.hpp b/vdi/hip_formatting.hpp index aab267d392..1e1c02b25f 100644 --- a/vdi/hip_formatting.hpp +++ b/vdi/hip_formatting.hpp @@ -17,8 +17,8 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "hip/hcc_detail/hiprtc.h" -#include "hip/hcc_detail/hip_runtime_prof.h" +#include +#include inline std::ostream& operator<<(std::ostream& os, const hipTextureFilterMode& s) { switch (s) { @@ -342,58 +342,6 @@ inline std::ostream& operator<<(std::ostream& os, const hiprtcResult& s) { return os; } -inline std::ostream& operator<<(std::ostream& os, const HipVdiOpId& s) { - switch (s) { - case kHipVdiOpIdDispatch: - os << "kHipVdiOpIdDispatch"; - break; - case kHipVdiOpIdCopy: - os << "kHipVdiOpIdCopy"; - break; - case kHipVdiOpIdBarrier: - os << "kHipVdiOpIdBarrier"; - break; - case kHipVdiOpIdNumber: - os << "kHipVdiOpIdNumber"; - break; - default: - os << "kHipVdiOpIdDispatch"; - }; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const HipVdiCommandKind& s) { - switch (s) { - case kHipVdiCommandKernel: - os << "kHipVdiCommandKernel"; - break; - case kHipVdiMemcpyDeviceToHost: - os << "kHipVdiMemcpyDeviceToHost"; - break; - case kHipHipVdiMemcpyHostToDevice: - os << "kHipHipVdiMemcpyHostToDevice"; - break; - case kHipVdiMemcpyDeviceToDevice: - os << "kHipVdiMemcpyDeviceToDevice"; - break; - case kHipVidMemcpyDeviceToHostRect: - os << "kHipVidMemcpyDeviceToHostRect"; - break; - case kHipVdiMemcpyHostToDeviceRect: - os << "kHipVdiMemcpyHostToDeviceRect"; - break; - case kHipVdiMemcpyDeviceToDeviceRect: - os << "kHipVdiMemcpyDeviceToDeviceRect"; - break; - case kHipVdiFillMemory: - os << "kHipVdiFillMemory"; - break; - default: - os << "kHipVdiCommandKernel"; - }; - return os; -} - inline std::ostream& operator<<(std::ostream& os, const hipJitOption& s) { switch (s) { case hipJitOptionMaxRegisters: From 9ecd0e79b6303665420b218b656032c0f6374a46 Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Thu, 12 Mar 2020 12:22:29 -0700 Subject: [PATCH 093/154] libhiprtc.so needs to be installed in DEST dir Change-Id: I3ae8b4f18a05a707b87fa73873c1721928ebe904 --- packaging/hip-vdi.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/hip-vdi.txt b/packaging/hip-vdi.txt index 48d0a61791..c80e4aed6f 100644 --- a/packaging/hip-vdi.txt +++ b/packaging/hip-vdi.txt @@ -3,9 +3,8 @@ project(hip_vdi) install(FILES @PROJECT_BINARY_DIR@/lib/libamdhip64.so DESTINATION lib) install(FILES @PROJECT_BINARY_DIR@/lib/libhip_hcc.so DESTINATION lib) -if(NOT @HIP_COMPILER@ STREQUAL "clang") - install(FILES @PROJECT_BINARY_DIR@/libhiprtc.so DESTINATION lib) -endif() +install(FILES @PROJECT_BINARY_DIR@/lib/libhiprtc.so DESTINATION lib) + install(FILES @PROJECT_BINARY_DIR@/.hipInfo DESTINATION lib) install(FILES @PROJECT_BINARY_DIR@/hip-config.cmake @PROJECT_BINARY_DIR@/hip-config-version.cmake DESTINATION lib/cmake/hip) install(FILES @hip_SOURCE_DIR@/packaging/hip-targets.cmake @hip_SOURCE_DIR@/packaging/hip-targets-release.cmake DESTINATION lib/cmake/hip) From bfcce529fa1b492809f9a817abeb04ea7f2632fe Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 13 Mar 2020 17:28:19 -0400 Subject: [PATCH 094/154] Enable simpleTexture3D test for VDI Change-Id: Ida65de6e1cae64ac97b624eaab1340877a6fba73 --- tests/src/texture/simpleTexture3D.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/src/texture/simpleTexture3D.cpp b/tests/src/texture/simpleTexture3D.cpp index 673e373dcf..5395fc5d69 100644 --- a/tests/src/texture/simpleTexture3D.cpp +++ b/tests/src/texture/simpleTexture3D.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ @@ -52,16 +52,16 @@ __global__ void simpleKernel3DArray(T* outputData, int height,int depth) { for (int i = 0; i < depth; i++) { - for (int j = 0; j < height; j++) { - for (int k = 0; k < width; k++) { - if(std::is_same::value) - outputData[i*width*height + j*width + k] = tex3D(texf, texf.textureObject, k, j, i); - else if(std::is_same::value) - outputData[i*width*height + j*width + k] = tex3D(texi, texi.textureObject, k, j, i); - else if(std::is_same::value) - outputData[i*width*height + j*width + k] = tex3D(texc, texc.textureObject, k, j, i); - } - } + for (int j = 0; j < height; j++) { + for (int k = 0; k < width; k++) { + if(std::is_same::value) + outputData[i*width*height + j*width + k] = tex3D(texf, k, j, i); + else if(std::is_same::value) + outputData[i*width*height + j*width + k] = tex3D(texi, k, j, i); + else if(std::is_same::value) + outputData[i*width*height + j*width + k] = tex3D(texc, k, j, i); + } + } } } @@ -84,10 +84,10 @@ void runTest(int width,int height,int depth,texture(); hipArray *arr; - HIPCHECK(hipMalloc3DArray(&arr, &channelDesc, make_hipExtent(width, height, depth), hipArrayCubemap)); + HIPCHECK(hipMalloc3DArray(&arr, &channelDesc, make_hipExtent(width, height, depth), hipArrayDefault)); hipMemcpy3DParms myparms = {0}; myparms.srcPos = make_hipPos(0,0,0); myparms.dstPos = make_hipPos(0,0,0); From 64cd527335a08c8806e7a7bd9bbcabe09f6ddb6e Mon Sep 17 00:00:00 2001 From: Sameer Sahasrabuddhe Date: Fri, 6 Mar 2020 11:47:21 +0530 Subject: [PATCH 095/154] SWDEV-204784: separate printf declaration for vdi/clang There are now two implementations of printf in HIP: 1. The implemenation for HCC is controlled by the HC_FEATURE_PRINTF macro, and it works only with the HCC compiler used in combination with the HCC runtime. 2. The implementation for hip-clang requires the VDI runtime, and is always enabled with that combination. Change-Id: Ibaeda7900ffe2ce602ca0094aafed0f1147ac2b6 --- bin/hipcc | 1 + include/hip/hcc_detail/device_functions.h | 9 +++++++++ include/hip/hcc_detail/hip_runtime.h | 10 +++++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/bin/hipcc b/bin/hipcc index 83774a996b..013179b49e 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -133,6 +133,7 @@ if (defined $HIP_RUNTIME and $HIP_RUNTIME eq "VDI" and !defined $HIP_VDI_HOME) { } else { $HIP_VDI_HOME = $HIP_PATH; # use HIP_PATH } + $HIPCXXFLAGS .= "-D__HIP_VDI__"; } if (defined $HIP_VDI_HOME) { diff --git a/include/hip/hcc_detail/device_functions.h b/include/hip/hcc_detail/device_functions.h index b96bbba07f..1be853bc99 100644 --- a/include/hip/hcc_detail/device_functions.h +++ b/include/hip/hcc_detail/device_functions.h @@ -33,6 +33,15 @@ THE SOFTWARE. #include #include #include + +#if __HIP_CLANG_ONLY__ +#if __HIP_VDI__ +extern "C" __device__ int printf(const char *fmt, ...); +#else +static inline __device__ void printf(const char* format, All... all) {} +#endif +#endif + /* Integer Intrinsics */ diff --git a/include/hip/hcc_detail/hip_runtime.h b/include/hip/hcc_detail/hip_runtime.h index e86611b213..c1ad5b2fe5 100644 --- a/include/hip/hcc_detail/hip_runtime.h +++ b/include/hip/hcc_detail/hip_runtime.h @@ -311,15 +311,19 @@ extern "C" __device__ void* __hip_free(void* ptr); static inline __device__ void* malloc(size_t size) { return __hip_malloc(size); } static inline __device__ void* free(void* ptr) { return __hip_free(ptr); } -#if defined(__HCC_ACCELERATOR__) && defined(HC_FEATURE_PRINTF) +// Declare printf only for the HCC compiler. hip-clang is handled in +// device_functions.h +#if __HCC_ACCELERATOR__ +#if HC_FEATURE_PRINTF template static inline __device__ void printf(const char* format, All... all) { hc::printf(format, all...); } -#elif defined(__HCC_ACCELERATOR__) || __HIP__ +#else template static inline __device__ void printf(const char* format, All... all) {} -#endif +#endif // HC_FEATURE_PRINTF +#endif // __HCC_ACCELERATOR__ #endif //__HCC_OR_HIP_CLANG__ From 2dac19754894601a798beb78694f618dbee7267a Mon Sep 17 00:00:00 2001 From: Jatin Date: Fri, 13 Mar 2020 12:06:21 +0530 Subject: [PATCH 096/154] Enable saxpy Test Change-Id: Iadb5f631e1cebaf016b1835510771b3b7fac0a55 --- tests/src/hiprtc/saxpy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/hiprtc/saxpy.cpp b/tests/src/hiprtc/saxpy.cpp index a08c1c2399..d063578757 100755 --- a/tests/src/hiprtc/saxpy.cpp +++ b/tests/src/hiprtc/saxpy.cpp @@ -20,7 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp LINK_OPTIONS hiprtc EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s ../test_common.cpp LINK_OPTIONS hiprtc EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ From d432dbfe20933775e8bfd578748adb38aa9dedf4 Mon Sep 17 00:00:00 2001 From: Tao Sang Date: Fri, 13 Mar 2020 23:05:43 -0400 Subject: [PATCH 097/154] Fix failure to get global variables Implement _ihipGetGlobalVar() and ihipGetGlobalVar() to get global variables. Change-Id: I442ab6712e12306c3316f114f5dc42f6daefaad9 --- src/hip_clang.cpp | 215 +++++++++++++++++++++++++++++++++++++++-- src/hip_hcc_internal.h | 12 +++ src/hip_module.cpp | 7 +- 3 files changed, 225 insertions(+), 9 deletions(-) diff --git a/src/hip_clang.cpp b/src/hip_clang.cpp index f17d2673c7..635aa8391f 100644 --- a/src/hip_clang.cpp +++ b/src/hip_clang.cpp @@ -157,16 +157,215 @@ extern "C" void __hipRegisterFunction( g_functions.insert(std::make_pair(hostFunction, std::move(functions))); } +static inline const char* hsa_strerror(hsa_status_t status) { + const char* str = nullptr; + if (hsa_status_string(status, &str) == HSA_STATUS_SUCCESS) { + return str; + } + return "Unknown error"; +} + +struct RegisteredVar { +public: + RegisteredVar(): size_(0), devicePtr_(nullptr) {} + ~RegisteredVar() {} + + static inline const char* hsa_strerror(hsa_status_t status) { + const char* str = nullptr; + if (hsa_status_string(status, &str) == HSA_STATUS_SUCCESS) { + return str; + } + return "Unknown error"; +} + +hipDeviceptr_t getdeviceptr() const { return devicePtr_; }; + size_t getvarsize() const { return size_; }; + + size_t size_; // Size of the variable + hipDeviceptr_t devicePtr_; //Device Memory Address of the variable. +}; + +struct DeviceVar { + void* shadowVptr; + std::string hostVar; + size_t size; + std::vector* modules; + std::vector rvars; + bool dyn_undef; +}; + +std::unordered_multimap g_vars; + +//The logic follows PlatformState::getGlobalVar in VDI RT +static DeviceVar* findVar(std::string hostVar, int deviceId, hipModule_t hmod) { + DeviceVar* dvar = nullptr; + if (hmod != nullptr) { + // If module is provided, then get the var only from that module + auto var_range = g_vars.equal_range(hostVar); + for (auto it = var_range.first; it != var_range.second; ++it) { + if ((*it->second.modules)[deviceId] == hmod) { + dvar = &(it->second); + break; + } + } + } else { + // If var count is < 2, return the var + if (g_vars.count(hostVar) < 2) { + auto it = g_vars.find(hostVar); + dvar = ((it == g_vars.end()) ? nullptr : &(it->second)); + } else { + // If var count is > 2, return the original var, + // if original var count != 1, return g_vars.end()/Invalid + size_t orig_global_count = 0; + auto var_range = g_vars.equal_range(hostVar); + for (auto it = var_range.first; it != var_range.second; ++it) { + // when dyn_undef is set, it is a shadow var + if (it->second.dyn_undef == false) { + ++orig_global_count; + dvar = &(it->second); + } + } + dvar = ((orig_global_count == 1) ? dvar : nullptr); + } + } + return dvar; +} + +hipError_t ihipGetGlobalVar(hipDeviceptr_t* dev_ptr, size_t* size_ptr, + const char* hostVar, hipModule_t hmod) { + GET_TLS(); + auto ctx = ihipGetTlsDefaultCtx(); + + if (!ctx) return hipErrorInvalidValue; + + auto device = ctx->getDevice(); + + if (!device) return hipErrorInvalidValue; + + ihipDevice_t* currentDevice = ihipGetDevice(device->_deviceId); + + if (!currentDevice) return hipErrorInvalidValue; + + int deviceId = device->_deviceId; + + DeviceVar* dvar = findVar(std::string(hostVar), deviceId, hmod); + if (dvar == nullptr) return hipErrorInvalidValue; + + if (dvar->rvars[deviceId].getdeviceptr() == nullptr) return hipErrorInvalidValue; + + *size_ptr = dvar->rvars[deviceId].getvarsize(); + *dev_ptr = dvar->rvars[deviceId].getdeviceptr(); + return hipSuccess; +} + +static bool createGlobalVarObj(const hsa_executable_t& hsaExecutable, const hsa_agent_t& hasAgent, + const char* global_name, void** device_pptr, size_t* bytes) { + hsa_status_t status = HSA_STATUS_SUCCESS; + hsa_symbol_kind_t sym_type; + hsa_executable_symbol_t global_symbol; + std::string buildLog; + + /* Find HSA Symbol by name */ + status = hsa_executable_get_symbol_by_name(hsaExecutable, global_name, &hasAgent, + &global_symbol); + if (status != HSA_STATUS_SUCCESS) { + buildLog += "Error: Failed to find the Symbol by Name: "; + buildLog += hsa_strerror(status); + tprintf(DB_FB, "createGlobalVarObj: %s\n", buildLog.c_str()); + return false; + } + + /* Find HSA Symbol Type */ + status = hsa_executable_symbol_get_info(global_symbol, HSA_EXECUTABLE_SYMBOL_INFO_TYPE, + &sym_type); + if (status != HSA_STATUS_SUCCESS) { + buildLog += "Error: Failed to find the Symbol Type : "; + buildLog += hsa_strerror(status); + tprintf(DB_FB, "createGlobalVarObj: %s\n", buildLog.c_str()); + return false; + } + + /* Make sure symbol type is VARIABLE */ + if (sym_type != HSA_SYMBOL_KIND_VARIABLE) { + buildLog += "Error: Symbol is not of type VARIABLE : "; + buildLog += hsa_strerror(status); + tprintf(DB_FB, "createGlobalVarObj: %s\n", buildLog.c_str()); + return false; + } + + /* Retrieve the size of the variable */ + status = hsa_executable_symbol_get_info(global_symbol, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_SIZE, bytes); + + if (status != HSA_STATUS_SUCCESS) { + buildLog += "Error: Failed to retrieve the Symbol Size : "; + buildLog += hsa_strerror(status); + tprintf(DB_FB, "createGlobalVarObj: %s\n", buildLog.c_str()); + return false; + } + + /* Find HSA Symbol Address */ + status = hsa_executable_symbol_get_info(global_symbol, + HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, device_pptr); + if (status != HSA_STATUS_SUCCESS) { + buildLog += "Error: Failed to find the Symbol Address : "; + buildLog += hsa_strerror(status); + tprintf(DB_FB, "createGlobalVarObj: %s\n", buildLog.c_str()); + return false; + } else { + tprintf(DB_FB, "createGlobalVarObj: var %s : device=%p, size=%zu\n", global_name, *device_pptr, *bytes); + } + + return true; +} + +// Registers a device-side global variable. +// For each global variable in device code, there is a corresponding shadow +// global variable in host code. The shadow host variable is used to keep +// track of the value of the device side global variable between kernel +// executions. +// The basic logic is taken from VDI RT, but there is much difference. extern "C" void __hipRegisterVar( - std::vector* modules, - char* hostVar, - char* deviceVar, - const char* deviceName, - int ext, - int size, - int constant, - int global) + std::vector* modules, // The device modules containing code object + char* var, // The shadow variable in host code + char* hostVar, // Variable name in host code + const char* deviceVar, // Variable name in device code + int ext, // Whether this variable is external + int size, // Size of the variable + int constant, // Whether this variable is constant + int global) // Unknown, always 0 { + HIP_INIT_API(__hipRegisterVar, modules, var, hostVar, deviceVar, ext, size, constant, global); + + DeviceVar dvar{var, std::string{ hostVar }, static_cast(size), modules, + std::vector{ g_deviceCnt }, false }; + + for (int deviceId = 0; deviceId < g_deviceCnt; deviceId++) { + auto device = ihipGetDevice(deviceId); + if(!device) { + continue; + } + hsa_executable_t& executable = (*modules)[deviceId]->executable; + hsa_agent_t& agent = g_allAgents[deviceId + 1]; + size_t bytes = 0; + hipDeviceptr_t devicePtr = nullptr; + + bool success = createGlobalVarObj(executable, agent, hostVar, &devicePtr, &bytes); + if(!success) { + return; + } + dvar.rvars[deviceId].devicePtr_ = devicePtr; + dvar.rvars[deviceId].size_ = bytes; + + hc::AmPointerInfo ptrInfo(nullptr, devicePtr, devicePtr, bytes, device->_acc, true, false); + hc::am_memtracker_add(devicePtr, ptrInfo); + + #if USE_APP_PTR_FOR_CTX + hc::am_memtracker_update(devicePtr, device->_deviceId, 0u, ihipGetTlsDefaultCtx()); + #else + hc::am_memtracker_update(devicePtr, device->_deviceId, 0u); + #endif + } + g_vars.insert(std::make_pair(std::string(hostVar), dvar)); } extern "C" void __hipUnregisterFatBinary(std::vector* modules) diff --git a/src/hip_hcc_internal.h b/src/hip_hcc_internal.h index ac63f49dba..8eec7292c2 100644 --- a/src/hip_hcc_internal.h +++ b/src/hip_hcc_internal.h @@ -982,6 +982,18 @@ hipStream_t ihipSyncAndResolveStream(hipStream_t, bool lockAcquired = 0); hipError_t ihipStreamSynchronize(TlsData *tls, hipStream_t stream); void ihipStreamCallbackHandler(ihipStreamCallback_t* cb); +/** + * @brief Copies the memory address and size of symbol @p symbolName + * + * @param[in] symbolName - Symbol on device + * @param[out] devPtr - Pointer to a pointer to the memory referred to by the symbol + * @param[out] size - Pointer to the size of the symbol + * @return #hipSuccess, #hipErrorNotInitialized, #hipErrorNotFound, #hipErrorInvalidValue + * + */ +hipError_t ihipGetGlobalVar(hipDeviceptr_t* dev_ptr, size_t* size_ptr, const char* hostVar, + hipModule_t hmod = nullptr); + // Stream printf functions: inline std::ostream& operator<<(std::ostream& os, const ihipStream_t& s) { os << "stream:"; diff --git a/src/hip_module.cpp b/src/hip_module.cpp index b527d6ccb0..b6fbc45866 100644 --- a/src/hip_module.cpp +++ b/src/hip_module.cpp @@ -879,7 +879,12 @@ namespace hip_impl { hipError_t agent_globals::read_agent_global_from_process(hipDeviceptr_t* dptr, size_t* bytes, const char* name) { - return impl->read_agent_global_from_process(dptr, bytes, name); + hipError_t result = impl->read_agent_global_from_process(dptr, bytes, name); + if(result != hipSuccess) { + // For Clang Compiler + Hcc Rt + result = ihipGetGlobalVar(dptr, bytes, name); + } + return result; } } // Namespace hip_impl. From cc134f7c580c2ef2b15733eb8dcf306d98c651ff Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 13 Mar 2020 17:35:25 -0400 Subject: [PATCH 098/154] Rework hipNormalizedFloatValueTex test This is currently so buggy that it causes a runtime crash on Nvidia platfrom... Disable the new version for hcc and vdi, header fixes are required for it to pass. Currently tex1D returns a char, when the actual sampled pixel value is a float, so the hi 3 bytes get lost. Change-Id: I8222a4d8d1d8b101eb43f3f8dfbe4818f885f8ea --- .../texture/hipNormalizedFloatValueTex.cpp | 112 ++++++++++-------- 1 file changed, 65 insertions(+), 47 deletions(-) diff --git a/tests/src/texture/hipNormalizedFloatValueTex.cpp b/tests/src/texture/hipNormalizedFloatValueTex.cpp index 04660ad70b..b4aa3e9c05 100644 --- a/tests/src/texture/hipNormalizedFloatValueTex.cpp +++ b/tests/src/texture/hipNormalizedFloatValueTex.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc + * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc hcc vdi * TEST: %t * HIT_END */ @@ -30,73 +30,92 @@ THE SOFTWARE. #define SIZE 10 static float getNormalizedValue(const float value, - const enum hipArray_Format texFormat) { - switch (texFormat) { - case HIP_AD_FORMAT_SIGNED_INT8: - return (value / SCHAR_MAX); - case HIP_AD_FORMAT_UNSIGNED_INT8: - return (value / UCHAR_MAX); - case HIP_AD_FORMAT_SIGNED_INT16: - return (value / SHRT_MAX); - case HIP_AD_FORMAT_UNSIGNED_INT16: - return (value / USHRT_MAX); - default: - return value; - } + const hipChannelFormatDesc& desc) { + if ((desc.x == 8) && (desc.f == hipChannelFormatKindSigned)) + return (value / SCHAR_MAX); + if ((desc.x == 8) && (desc.f == hipChannelFormatKindUnsigned)) + return (value / UCHAR_MAX); + if ((desc.x == 16) && (desc.f == hipChannelFormatKindSigned)) + return (value / SHRT_MAX); + if ((desc.x == 16) && (desc.f == hipChannelFormatKindUnsigned)) + return (value / USHRT_MAX); + return value; } #if __HIP__ __hip_pinned_shadow__ #endif -texture textureNormalizedVal_1D; +texture texc; +#if __HIP__ +__hip_pinned_shadow__ +#endif +texture texuc; + +#if __HIP__ +__hip_pinned_shadow__ +#endif +texture texs; + +#if __HIP__ +__hip_pinned_shadow__ +#endif +texture texus; + + +template __global__ void normalizedValTextureTest(unsigned int numElements, float* pDst) { unsigned int elementID = hipThreadIdx_x; if(elementID >= numElements) - return; + return; float coord =(float) elementID/numElements; - pDst[elementID] = tex1D(textureNormalizedVal_1D, coord); + if(std::is_same::value) + pDst[elementID] = tex1D(texc, coord); + else if(std::is_same::value) + pDst[elementID] = tex1D(texuc, coord); + else if(std::is_same::value) + pDst[elementID] = tex1D(texs, coord); + else if(std::is_same::value) + pDst[elementID] = tex1D(texus, coord); } template -bool textureTest(enum hipArray_Format texFormat) +bool textureTest(texture *tex) { + hipChannelFormatDesc desc = hipCreateChannelDesc(); + hipArray_t dData; + HIPCHECK(hipMallocArray(&dData, &desc, SIZE, 1, hipArrayDefault)); + T hData[] = {65, 66, 67, 68, 69, 70, 71, 72, 73, 74}; - T *dData = NULL; - HIPCHECK(hipMalloc((void **) &dData, sizeof(T)*SIZE)); - HIPCHECK(hipMemcpyHtoD((hipDeviceptr_t)dData, hData, sizeof(T)*SIZE)); - textureReference* texRef = &textureNormalizedVal_1D; - HIPCHECK(hipTexRefSetAddressMode(texRef, 0, hipAddressModeClamp)); - HIPCHECK(hipTexRefSetAddressMode(texRef, 1, hipAddressModeClamp)); - HIPCHECK(hipTexRefSetFilterMode(texRef, hipFilterModePoint)); - HIPCHECK(hipTexRefSetFlags(texRef, HIP_TRSF_NORMALIZED_COORDINATES)); - HIPCHECK(hipTexRefSetFormat(texRef, texFormat, 1)); - - size_t offSet = 0; - HIPCHECK(hipTexRefSetAddress(&offSet, texRef, (hipDeviceptr_t)dData, sizeof(T)*SIZE)); - - bool testResult = true; + HIPCHECK(hipMemcpy2DToArray(dData, 0, 0, hData, sizeof(T)*SIZE, sizeof(T)*SIZE, 1, hipMemcpyHostToDevice)); + + tex->normalized = true; + tex->channelDesc = desc; + HIPCHECK(hipBindTextureToArray(tex, dData, &desc)); + float *dOutputData = NULL; HIPCHECK(hipMalloc((void **) &dOutputData, sizeof(float)*SIZE)); - - hipLaunchKernelGGL(HIP_KERNEL_NAME(normalizedValTextureTest), dim3(1,1,1), dim3(SIZE,1,1), 0, 0, SIZE, dOutputData); + + hipLaunchKernelGGL(normalizedValTextureTest, dim3(1,1,1), dim3(SIZE,1,1), 0, 0, SIZE, dOutputData); float *hOutputData = new float[SIZE]; - HIPCHECK(hipMemcpyDtoH(hOutputData, (hipDeviceptr_t)dOutputData, (sizeof(float)*SIZE))); - + HIPCHECK(hipMemcpy(hOutputData, dOutputData, (sizeof(float)*SIZE), hipMemcpyDeviceToHost)); + + bool testResult = true; for(int i = 0; i < SIZE; i++) { - float expected = getNormalizedValue(float(hData[i]), texFormat); + float expected = getNormalizedValue(float(hData[i]), desc); if(expected != hOutputData[i]) { - printf("mismatch at index:%d for texType:%d output:%f\n",i,texFormat,hOutputData[i]); + printf("mismatch at index:%d output:%f expected:%f\n",i,hOutputData[i],expected); testResult = false; - break; + break; } } - hipFree(dData); - hipFree(dOutputData); + + HIPCHECK(hipFreeArray(dData)); + HIPCHECK(hipFree(dOutputData)); delete [] hOutputData; return testResult; } @@ -113,12 +132,11 @@ int main(int argc, char** argv) std::cout << "Arch - AMD GPU :: " << props.gcnArch << std::endl; #endif - status &= textureTest (HIP_AD_FORMAT_SIGNED_INT8); - status &= textureTest (HIP_AD_FORMAT_UNSIGNED_INT8); - status &= textureTest (HIP_AD_FORMAT_SIGNED_INT16); - status &= textureTest(HIP_AD_FORMAT_UNSIGNED_INT16); - status &= textureTest (HIP_AD_FORMAT_FLOAT); - + status &= textureTest (&texc); + status &= textureTest (&texuc); + status &= textureTest (&texs); + status &= textureTest(&texus); + if(status){ passed(); } From a0751402d8296ce7cc1254dae4902801b7d284ee Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 13 Mar 2020 18:28:14 -0400 Subject: [PATCH 099/154] Rework device texture headers This change addresses three things. First the available APIs are brought up to par with Cuda (missing ones are added and incorrect ones removed). Second the size of hip/hcc_detail/texture_functions.h. Using some template magic we can bring down the code size down from ~11k lines to only ~900 lines in total. Third this change fixes some bugs in the declaration of the texture fetch funcitons. Currently the return type for textures with read mode set to hipReadModeNormalizedFloat is not float. This causes pixel data to be lost during the bitcast when the texture pixel element size is less than the size of float. The new headers will only be enabled for VDI to avoid breaking HCC. Change-Id: I77cb29293fb79e55681be094c37702a48d80b64c --- include/hip/hcc_detail/hip_runtime.h | 5 +- include/hip/hcc_detail/ockl_image.h | 133 ++++++ .../hip/hcc_detail/texture_fetch_functions.h | 272 +++++++++++++ .../hcc_detail/texture_indirect_functions.h | 382 ++++++++++++++++++ 4 files changed, 791 insertions(+), 1 deletion(-) create mode 100644 include/hip/hcc_detail/ockl_image.h create mode 100644 include/hip/hcc_detail/texture_fetch_functions.h create mode 100644 include/hip/hcc_detail/texture_indirect_functions.h diff --git a/include/hip/hcc_detail/hip_runtime.h b/include/hip/hcc_detail/hip_runtime.h index c1ad5b2fe5..cbb3a0e99a 100644 --- a/include/hip/hcc_detail/hip_runtime.h +++ b/include/hip/hcc_detail/hip_runtime.h @@ -103,9 +103,12 @@ extern int HIP_TRACE_API; #include #include #include -#include #if __HCC__ #include + #include +#else + #include + #include #endif // TODO-HCC remove old definitions ; ~1602 hcc supports __HCC_ACCELERATOR__ define. #if defined(__KALMAR_ACCELERATOR__) && !defined(__HCC_ACCELERATOR__) diff --git a/include/hip/hcc_detail/ockl_image.h b/include/hip/hcc_detail/ockl_image.h new file mode 100644 index 0000000000..d9bc296791 --- /dev/null +++ b/include/hip/hcc_detail/ockl_image.h @@ -0,0 +1,133 @@ +/* +Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#pragma once + +#include + +extern "C" { + +#define ADDRESS_SPACE_CONSTANT __attribute__((address_space(4))) + +__device__ float4::Native_vec_ __ockl_image_load_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c); + +__device__ float4::Native_vec_ __ockl_image_load_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_load_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_load_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_load_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_load_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f); + +__device__ float4::Native_vec_ __ockl_image_load_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f); + +__device__ float4::Native_vec_ __ockl_image_load_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l); + +__device__ float4::Native_vec_ __ockl_image_load_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l); + +__device__ float4::Native_vec_ __ockl_image_load_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l); + +__device__ float4::Native_vec_ __ockl_image_load_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l); + +__device__ float4::Native_vec_ __ockl_image_load_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l); + +__device__ float4::Native_vec_ __ockl_image_load_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, int l); + +__device__ float4::Native_vec_ __ockl_image_load_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, int l); + +__device__ void __ockl_image_store_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, float4::Native_vec_ p); + +__device__ void __ockl_image_store_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p); + +__device__ void __ockl_image_store_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p); + +__device__ void __ockl_image_store_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p); + +__device__ void __ockl_image_store_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p); + +__device__ void __ockl_image_store_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p); + +__device__ void __ockl_image_store_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p); + +__device__ void __ockl_image_store_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l, float4::Native_vec_ p); + +__device__ void __ockl_image_store_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p); + +__device__ void __ockl_image_store_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p); + +__device__ void __ockl_image_store_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p); + +__device__ void __ockl_image_store_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p); + +__device__ void __ockl_image_store_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p); + +__device__ void __ockl_image_store_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p); + +__device__ float4::Native_vec_ __ockl_image_sample_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c); + +__device__ float4::Native_vec_ __ockl_image_sample_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_sample_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_sample_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_sample_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_sample_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_sample_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_sample_grad_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float dx, float dy); + +__device__ float4::Native_vec_ __ockl_image_sample_grad_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float dx, float dy); + +__device__ float4::Native_vec_ __ockl_image_sample_grad_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy); + +__device__ float4::Native_vec_ __ockl_image_sample_grad_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy); + +__device__ float4::Native_vec_ __ockl_image_sample_grad_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float4::Native_vec_ dx, float4::Native_vec_ dy); + +__device__ float4::Native_vec_ __ockl_image_sample_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float l); + +__device__ float4::Native_vec_ __ockl_image_sample_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l); + +__device__ float4::Native_vec_ __ockl_image_sample_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l); + +__device__ float4::Native_vec_ __ockl_image_sample_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l); + +__device__ float4::Native_vec_ __ockl_image_sample_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l); + +__device__ float4::Native_vec_ __ockl_image_sample_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l); + +__device__ float4::Native_vec_ __ockl_image_sample_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l); + +__device__ float4::Native_vec_ __ockl_image_gather4r_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_gather4g_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_gather4b_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); + +__device__ float4::Native_vec_ __ockl_image_gather4a_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c); + +}; \ No newline at end of file diff --git a/include/hip/hcc_detail/texture_fetch_functions.h b/include/hip/hcc_detail/texture_fetch_functions.h new file mode 100644 index 0000000000..ad8deda400 --- /dev/null +++ b/include/hip/hcc_detail/texture_fetch_functions.h @@ -0,0 +1,272 @@ +/* +Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#pragma once + +#if defined(__cplusplus) + +#include +#include +#include + +#include + +#define TEXTURE_PARAMETERS_INIT \ + unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)t.textureObject; \ + unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; + +template +struct __hip_tex_ret {}; + +template +using __hip_tex_ret_t = typename __hip_tex_ret::type; + +template +struct __hip_tex_ret { using type = T; }; + +template +struct __hip_tex_ret { using type = float; }; + +template +struct __hip_tex_ret, hipReadModeNormalizedFloat> { using type = HIP_vector_type; }; + +template +static __forceinline__ __device__ __hip_tex_ret_t tex1Dfetch(texture t, int x) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_1D(i, s, x); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex1D(texture t, float x) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_1D(i, s, x); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex2D(texture t, float x, float y) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_2D(i, s, float2(x, y).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex1DLayered(texture t, float x, int layer) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex2DLayered(texture t, float x, float y, int layer) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex3D(texture t, float x, float y, float z) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t texCubemap(texture t, float x, float y, float z) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_CM(i, s, float4(x, y, z, 0.0f).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex1DLod(texture t, float x, float level) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_lod_1D(i, s, x, level); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex2DLod(texture t, float x, float y, float level) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex1DLayeredLod(texture t, float x, int layer, float level) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex2DLayeredLod(texture t, float x, float y, int layer, float level) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_lod_2Da(i, s, float4(x, y, layer, 0.0f).data, level); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex3DLod(texture t, float x, float y, float z, float level) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, level); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t texCubemapLod(texture t, float x, float y, float z, float level) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_lod_CM(i, s, float4(x, y, z, 0.0f).data, level); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayered(texture t, float x, float y, float z, int layer) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_CMa(i, s, float4(x, y, z, layer).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayeredLod(texture t, float x, float y, float z, int layer, float level) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_lod_CMa(i, s, float4(x, y, z, layer).data, level); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t texCubemapGrad(texture t, float x, float y, float z, float4 dPdx, float4 dPdy) +{ + TEXTURE_PARAMETERS_INIT; + // TODO missing in device libs. + // auto tmp = __ockl_image_sample_grad_CM(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); + // return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); + return {}; +} + +template +static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayeredGrad(texture t, float x, float y, float z, int layer, float4 dPdx, float4 dPdy) +{ + TEXTURE_PARAMETERS_INIT; + // TODO missing in device libs. + // auto tmp = __ockl_image_sample_grad_CMa(i, s, float4(x, y, z, layer).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); + // return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); + return {}; +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex1DGrad(texture t, float x, float dPdx, float dPdy) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex2DGrad(texture t, float x, float y, float2 dPdx, float2 dPdy) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex1DLayeredGrad(texture t, float x, int layer, float dPdx, float dPdy) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dPdx, dPdy); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex2DLayeredGrad(texture t, float x, float y, int layer, float2 dPdx, float2 dPdy) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +static __forceinline__ __device__ __hip_tex_ret_t tex3DGrad(texture t, float x, float y, float z, float4 dPdx, float4 dPdy) +{ + TEXTURE_PARAMETERS_INIT; + auto tmp = __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); + return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); +} + +template +struct __hip_tex2dgather_ret {}; + +template +using __hip_tex2dgather_ret_t = typename __hip_tex2dgather_ret::type; + +template +struct __hip_tex2dgather_ret { using type = HIP_vector_type; }; + +template +struct __hip_tex2dgather_ret, hipReadModeElementType> { using type = HIP_vector_type; }; + +template +struct __hip_tex2dgather_ret { using type = float4; }; + +template +static __forceinline__ __device__ __hip_tex2dgather_ret_t tex2Dgather(texture t, float x, float y, int comp=0) +{ + TEXTURE_PARAMETERS_INIT; + switch (comp) { + case 1: { + auto tmp = __ockl_image_gather4g_2D(i, s, float2(x, y).data); + return *reinterpret_cast<__hip_tex2dgather_ret_t*>(&tmp); + } + case 2: { + auto tmp = __ockl_image_gather4b_2D(i, s, float2(x, y).data); + return *reinterpret_cast<__hip_tex2dgather_ret_t*>(&tmp); + } + case 3: { + auto tmp = __ockl_image_gather4a_2D(i, s, float2(x, y).data); + return *reinterpret_cast<__hip_tex2dgather_ret_t*>(&tmp); + } + default: { + auto tmp = __ockl_image_gather4r_2D(i, s, float2(x, y).data); + return *reinterpret_cast<__hip_tex2dgather_ret_t*>(&tmp); + } + } + return {}; +} + +#endif diff --git a/include/hip/hcc_detail/texture_indirect_functions.h b/include/hip/hcc_detail/texture_indirect_functions.h new file mode 100644 index 0000000000..56784f433a --- /dev/null +++ b/include/hip/hcc_detail/texture_indirect_functions.h @@ -0,0 +1,382 @@ +/* +Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#pragma once + +#if defined(__cplusplus) + +#include +#include +#include + +#define TEXTURE_OBJECT_PARAMETERS_INIT \ + unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)textureObject; \ + unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; + +template +static __device__ T tex1Dfetch(hipTextureObject_t textureObject, int x) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_1D(i, s, x); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex1Dfetch(T *ptr, hipTextureObject_t textureObject, int x) +{ + *ptr = tex1Dfetch(textureObject, x); +} + +template +static __device__ T tex1D(hipTextureObject_t textureObject, float x) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_1D(i, s, x); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex1D(T *ptr, hipTextureObject_t textureObject, float x) +{ + *ptr = tex1D(textureObject, x); +} + +template +static __device__ T tex2D(hipTextureObject_t textureObject, float x, float y) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_2D(i, s, float2(x, y).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex2D(T *ptr, hipTextureObject_t textureObject, float x, float y) +{ + *ptr = tex2D(textureObject, x, y); +} + +template +static __device__ T tex3D(hipTextureObject_t textureObject, float x, float y, float z) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex3D(T *ptr, hipTextureObject_t textureObject, float x, float y, float z) +{ + *ptr = tex3D(textureObject, x, y, z); +} + +template +static __device__ T tex1DLayered(hipTextureObject_t textureObject, float x, int layer) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex1DLayered(T *ptr, hipTextureObject_t textureObject, float x, int layer) +{ + *ptr = tex1DLayered(textureObject, x, layer); +} + +template +static __device__ T tex2DLayered(hipTextureObject_t textureObject, float x, float y, int layer) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex2DLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer) +{ + *ptr = tex1DLayered(textureObject, x, y, layer); +} + +template +static __device__ T texCubemap(hipTextureObject_t textureObject, float x, float y, float z) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_CM(i, s, float4(x, y, z, 0.0f).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void texCubemap(T *ptr, hipTextureObject_t textureObject, float x, float y, float z) +{ + *ptr = texCubemap(textureObject, x, y, z); +} + +template +static __device__ T texCubemapLayered(hipTextureObject_t textureObject, float x, float y, float z, int layer) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_CMa(i, s, float4(x, y, z, layer).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void texCubemapLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer) +{ + *ptr = texCubemapLayered(textureObject, x, y, z, layer); +} + +template +static __device__ T tex2Dgather(hipTextureObject_t textureObject, float x, float y, int comp = 0) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + switch (comp) { + case 1: { + auto tmp = __ockl_image_gather4r_2D(i, s, float2(x, y).data); + return *reinterpret_cast(&tmp); + break; + } + case 2: { + auto tmp = __ockl_image_gather4g_2D(i, s, float2(x, y).data); + return *reinterpret_cast(&tmp); + break; + } + case 3: { + auto tmp = __ockl_image_gather4b_2D(i, s, float2(x, y).data); + return *reinterpret_cast(&tmp); + break; + } + default: { + auto tmp = __ockl_image_gather4a_2D(i, s, float2(x, y).data); + return *reinterpret_cast(&tmp); + break; + } + }; + return {}; +} + +template +static __device__ void tex2Dgather(T *ptr, hipTextureObject_t textureObject, float x, float y, int comp = 0) +{ + *ptr = texCubemapLayered(textureObject, x, y, comp); +} + +template +static __device__ T tex1DLod(hipTextureObject_t textureObject, float x, float level) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_lod_1D(i, s, x, level); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex1DLod(T *ptr, hipTextureObject_t textureObject, float x, float level) +{ + *ptr = tex1DLod(textureObject, x, level); +} + +template +static __device__ T tex2DLod(hipTextureObject_t textureObject, float x, float y, float level) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex2DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float level) +{ + *ptr = tex2DLod(textureObject, x, y, level); +} + +template +static __device__ T tex3DLod(hipTextureObject_t textureObject, float x, float y, float z, float level) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, level); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex3DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level) +{ + *ptr = tex3DLod(textureObject, x, y, z, level); +} + +template +static __device__ T tex1DLayeredLod(hipTextureObject_t textureObject, float x, int layer, float level) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex1DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, int layer, float level) +{ + *ptr = tex1DLayeredLod(textureObject, x, layer, level); +} + +template +static __device__ T tex2DLayeredLod(hipTextureObject_t textureObject, float x, float y, int layer, float level) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex2DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float level) +{ + *ptr = tex2DLayeredLod(textureObject, x, y, layer, level); +} + +template +static __device__ T texCubemapLod(hipTextureObject_t textureObject, float x, float y, float z, float level) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_lod_CM(i, s, float4(x, y, z, 0.0f).data, level); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void texCubemapLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level) +{ + *ptr = texCubemapLod(textureObject, x, y, z, level); +} + +template +static __device__ T texCubemapGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + // TODO missing in device libs. + // auto tmp = __ockl_image_sample_grad_CM(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); + // return *reinterpret_cast(&tmp); + return {}; +} + +template +static __device__ void texCubemapGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) +{ + *ptr = texCubemapGrad(textureObject, x, y, z, dPdx, dPdy); +} + +template +static __device__ T texCubemapLayeredLod(hipTextureObject_t textureObject, float x, float y, float z, int layer, float level) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_lod_CMa(i, s, float4(x, y, z, layer).data, level); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void texCubemapLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float level) +{ + *ptr = texCubemapLayeredLod(textureObject, x, y, z, layer, level); +} + +template +static __device__ T tex1DGrad(hipTextureObject_t textureObject, float x, float dPdx, float dPdy) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex1DGrad(T *ptr, hipTextureObject_t textureObject, float x, float dPdx, float dPdy) +{ + *ptr = tex1DGrad(textureObject, x, dPdx, dPdy); +} + +template +static __device__ T tex2DGrad(hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex2DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy) +{ + *ptr = tex2DGrad(textureObject, x, y, dPdx, dPdy); +} + +template +static __device__ T tex3DGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex3DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) +{ + *ptr = tex3DGrad(textureObject, x, y, z, dPdx, dPdy); +} + +template +static __device__ T tex1DLayeredGrad(hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dPdx, dPdy); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex1DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy) +{ + *ptr = tex1DLayeredGrad(textureObject, x, layer, dPdx, dPdy); +} + +template +static __device__ T tex2DLayeredGrad(hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + auto tmp = __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); + return *reinterpret_cast(&tmp); +} + +template +static __device__ void tex2DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy) +{ + *ptr = tex2DLayeredGrad(textureObject, x, y, layer, dPdx, dPdy); +} + +template +static __device__ T texCubemapLayeredGrad(hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy) +{ + TEXTURE_OBJECT_PARAMETERS_INIT + // TODO missing in device libs. + // auto tmp = __ockl_image_sample_grad_CMa(i, s, float4(x, y, z, layer).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data); + // return *reinterpret_cast(&tmp); + return {}; +} + +template +static __device__ void texCubemapLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy) +{ + *ptr = texCubemapLayeredGrad(textureObject, x, y, z, layer, dPdx, dPdy); +} + +#endif From a605458a0cefe659925dd9f2604f6db9b77f0488 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 13 Mar 2020 18:33:52 -0400 Subject: [PATCH 100/154] Enable hipNormalizedFloatValueTex test for VDI Change-Id: Iac8631312f34821d919b69b3fcb92b9387ba31dd --- tests/src/texture/hipNormalizedFloatValueTex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/texture/hipNormalizedFloatValueTex.cpp b/tests/src/texture/hipNormalizedFloatValueTex.cpp index b4aa3e9c05..96917ecb62 100644 --- a/tests/src/texture/hipNormalizedFloatValueTex.cpp +++ b/tests/src/texture/hipNormalizedFloatValueTex.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc hcc vdi + * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc hcc * TEST: %t * HIT_END */ From 1b288456ae9c430c634c3a84e16ee24007f0f6dc Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 12:27:10 -0400 Subject: [PATCH 101/154] Enable simpleTexture2DLayered test for VDI Change-Id: I420f68824c6825152ac50e5a129b11b6ad88deb9 --- tests/src/texture/simpleTexture2DLayered.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/texture/simpleTexture2DLayered.cpp b/tests/src/texture/simpleTexture2DLayered.cpp index f4d3aac1e5..e5014dae6b 100644 --- a/tests/src/texture/simpleTexture2DLayered.cpp +++ b/tests/src/texture/simpleTexture2DLayered.cpp @@ -21,7 +21,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp * TEST: %t * HIT_END */ From 82926666c4641b9d2d9f38611cbb0d7c5267926a Mon Sep 17 00:00:00 2001 From: Sarbojit Sarkar Date: Wed, 11 Mar 2020 19:06:22 +0530 Subject: [PATCH 102/154] [hip-vdi]Fix for TF build failure [SWDEV-225827] Change-Id: I8478779bef92bad8353b8d066b28c220bb59b98d --- include/hip/hcc_detail/hip_runtime_api.h | 7 +++ .../hipOccupancyMaxPotentialBlockSize.cpp | 4 +- vdi/hip_hcc.def.in | 1 + vdi/hip_hcc.map.in | 1 + vdi/hip_platform.cpp | 48 ++++++++++++++----- 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 2360333b1a..df27b851c4 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -3437,6 +3437,13 @@ hipError_t hipBindTextureToMipmappedArray(const texture& tex, } #if __HIP_VDI__ && !defined(__HCC__) + +template +inline hipError_t hipOccupancyMaxPotentialBlockSize(uint32_t* gridSize, uint32_t* blockSize, + F kernel, size_t dynSharedMemPerBlk, uint32_t blockSizeLimit) { +return hipOccupancyMaxPotentialBlockSize(gridSize, blockSize,(hipFunction_t)kernel, dynSharedMemPerBlk, blockSizeLimit); +} + template inline hipError_t hipLaunchCooperativeKernel(T f, dim3 gridDim, dim3 blockDim, void** kernelParams, unsigned int sharedMemBytes, hipStream_t stream) { diff --git a/tests/src/runtimeApi/module/hipOccupancyMaxPotentialBlockSize.cpp b/tests/src/runtimeApi/module/hipOccupancyMaxPotentialBlockSize.cpp index 8a613bf77e..a81862952d 100644 --- a/tests/src/runtimeApi/module/hipOccupancyMaxPotentialBlockSize.cpp +++ b/tests/src/runtimeApi/module/hipOccupancyMaxPotentialBlockSize.cpp @@ -22,7 +22,7 @@ THE SOFTWARE. // Test the Grid_Launch syntax. /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ @@ -46,10 +46,8 @@ int main(int argc, char* argv[]) { // test case for using kernel function pointer uint32_t gridSize = 0; uint32_t blockSize = 0; -#if defined(__HIP_PLATFORM_HCC__) && GENERIC_GRID_LAUNCH == 1 && defined(__HCC__) hipOccupancyMaxPotentialBlockSize(&gridSize, &blockSize, f1, 0, 0); assert(gridSize != 0 && blockSize != 0); -#endif // test case for using kernel function pointer with template gridSize = 0; diff --git a/vdi/hip_hcc.def.in b/vdi/hip_hcc.def.in index 4ec2f698c0..68fc2cd13a 100644 --- a/vdi/hip_hcc.def.in +++ b/vdi/hip_hcc.def.in @@ -135,6 +135,7 @@ hipModuleLoad hipModuleLoadData hipModuleLoadDataEx hipModuleUnload +hipOccupancyMaxPotentialBlockSize hipOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags hipFuncGetAttributes diff --git a/vdi/hip_hcc.map.in b/vdi/hip_hcc.map.in index b8ee45d647..aafc81f4be 100644 --- a/vdi/hip_hcc.map.in +++ b/vdi/hip_hcc.map.in @@ -135,6 +135,7 @@ global: hipModuleLoadData; hipModuleLoadDataEx; hipModuleUnload; + hipOccupancyMaxPotentialBlockSize; hipOccupancyMaxActiveBlocksPerMultiprocessor; hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags; hipFuncGetAttributes; diff --git a/vdi/hip_platform.cpp b/vdi/hip_platform.cpp index 3dd5c97089..57a37b02a5 100644 --- a/vdi/hip_platform.cpp +++ b/vdi/hip_platform.cpp @@ -604,13 +604,12 @@ hipError_t ihipCreateGlobalVarObj(const char* name, hipModule_t hmod, amd::Memor namespace hip_impl { - -hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, - hipFunction_t f, - int blockSize, - size_t dynamicSMemSize) +hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, int* numGrids, + hipFunction_t f, int blockSize, + size_t dynamicSMemSize, bool bCalcPotentialBlkSz) { - HIP_INIT_API(NONE, f, blockSize, dynamicSMemSize); + HIP_INIT_API(NONE, f, blockSize, dynamicSMemSize, bCalcPotentialBlkSz); + if(numBlocks == nullptr){HIP_RETURN(hipErrorInvalidValue);} int deviceId = ihipGetDevice(); // FIXME: Function may not be a device function and may have been obtaiend via // hipModuleGetFunction and thus not in the functions_ map. Check the map @@ -627,12 +626,16 @@ hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, if (!kernel) { HIP_RETURN(hipErrorOutOfMemory); } - if (blockSize == 0) { - HIP_RETURN(hipErrorInvalidValue); - } amd::Device* device = hip::getCurrentDevice()->devices()[0]; const device::Kernel::WorkGroupInfo* wrkGrpInfo = kernel->getDeviceKernel(*device)->workGroupInfo(); - + if (blockSize == 0) { + if (bCalcPotentialBlkSz == false){ + HIP_RETURN(hipErrorInvalidValue); + } + else { + blockSize = device->info().maxWorkGroupSize_; // maxwavefrontperblock + } + } // Find threads accupancy per CU => simd_per_cu * GPR usage constexpr size_t MaxWavesPerSimd = 8; // Limited by SPI 32 per CU, hence 8 per SIMD size_t VgprWaves = wrkGrpInfo->availableVGPRs_ / amd::alignUp(wrkGrpInfo->usedVGPRs_, 4); @@ -658,12 +661,33 @@ hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, int lds_occupancy = static_cast(device->info().localMemSize_ / total_used_lds); *numBlocks = std::min(*numBlocks, lds_occupancy); } + if (bCalcPotentialBlkSz){ + if (numGrids == nullptr){ + HIP_RETURN(hipErrorInvalidValue); + } + *numGrids = *numBlocks * device->info().numRTCUs_; + } HIP_RETURN(hipSuccess); } } extern "C" { +// FIXME: Need to replace `uint32_t` with `int` finally. +hipError_t hipOccupancyMaxPotentialBlockSize(uint32_t* gridSize, uint32_t* blockSize, + hipFunction_t f, size_t dynSharedMemPerBlk, + uint32_t blockSizeLimit) +{ + int numGrids = 0; + int numBlocks = 0; + hipError_t Ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(&numBlocks, &numGrids, f, 0, dynSharedMemPerBlk,true); + if (Ret == hipSuccess){ + *blockSize = numBlocks; + *gridSize = numGrids; + } + HIP_RETURN(Ret); +} + // FIXME: Need to replace `uint32_t` with `int` finally. hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor(uint32_t* numBlocks, hipFunction_t f, @@ -671,7 +695,7 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor(uint32_t* numBlocks, size_t dynamicSMemSize) { int NB; - hipError_t Ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(&NB, f, blockSize, dynamicSMemSize); + hipError_t Ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(&NB, nullptr, f, blockSize, dynamicSMemSize, false); *numBlocks = NB; HIP_RETURN(Ret); } @@ -684,7 +708,7 @@ hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(uint32_t* numBl unsigned int flags) { int NB; - hipError_t Ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(&NB, f, blockSize, dynamicSMemSize); + hipError_t Ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(&NB, nullptr, f, blockSize, dynamicSMemSize, false); *numBlocks = NB; HIP_RETURN(Ret); } From ef2415edc7a3bce96d1fda27d90e3fb4525881a7 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 12:52:20 -0400 Subject: [PATCH 103/154] Add constraints to texture fetch functions When sampling a pixel the hw always returns a float4. The type in the texture reference controls the bitcast that we perform before returning the sampled pixel. Creating a texture with an unsupported will lead to potential UB. This change makes it so that it's only possible to use textures with a type that makes sense. Using something like texture will now lead to a compilation error with a message "Invalid channel type!". Change-Id: I7fde44cb1d4b9737e0c48c28cb59c018c59ccaa2 --- .../hip/hcc_detail/texture_fetch_functions.h | 150 +++++++++++++++--- 1 file changed, 132 insertions(+), 18 deletions(-) diff --git a/include/hip/hcc_detail/texture_fetch_functions.h b/include/hip/hcc_detail/texture_fetch_functions.h index ad8deda400..39ac3ecd89 100644 --- a/include/hip/hcc_detail/texture_fetch_functions.h +++ b/include/hip/hcc_detail/texture_fetch_functions.h @@ -34,20 +34,106 @@ THE SOFTWARE. unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)t.textureObject; \ unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; -template -struct __hip_tex_ret {}; +template +struct __hip_is_channel_type +{ + static constexpr bool value = + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value; +}; -template -using __hip_tex_ret_t = typename __hip_tex_ret::type; - -template -struct __hip_tex_ret { using type = T; }; +template< + typename T, + unsigned int rank> +struct __hip_is_channel_type> +{ + static constexpr bool value = + __hip_is_channel_type::value && + ((rank == 1) || + (rank == 2) || + (rank == 4)); +}; template -struct __hip_tex_ret { using type = float; }; +struct __hip_is_normalized_channel_type +{ + static constexpr bool value = + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value; +}; -template -struct __hip_tex_ret, hipReadModeNormalizedFloat> { using type = HIP_vector_type; }; +template< + typename T, + unsigned int rank> +struct __hip_is_normalized_channel_type> +{ + static constexpr bool value = + __hip_is_normalized_channel_type::value && + ((rank == 1) || + (rank == 2) || + (rank == 4)); +}; + +template < + typename T, + hipTextureReadMode readMode, + typename Enable = void> +struct __hip_tex_ret +{ + static_assert(std::is_same::value, "Invalid channel type!"); +}; + +template < + typename T, + hipTextureReadMode readMode> +using __hip_tex_ret_t = typename __hip_tex_ret::type; + +template +struct __hip_tex_ret< + T, + hipReadModeElementType, + typename std::enable_if<__hip_is_channel_type::value, bool>::type> +{ + using type = T; +}; + +template< + typename T, + unsigned int rank> +struct __hip_tex_ret< + HIP_vector_type, + hipReadModeElementType, + typename std::enable_if<__hip_is_channel_type>::value, bool>::type> +{ + using type = HIP_vector_type<__hip_tex_ret_t, rank>; +}; + +template +struct __hip_tex_ret< + T, + hipReadModeNormalizedFloat, + typename std::enable_if<__hip_is_normalized_channel_type::value, bool>::type> +{ + using type = float; +}; + +template< + typename T, + unsigned int rank> +struct __hip_tex_ret< + HIP_vector_type, + hipReadModeNormalizedFloat, + typename std::enable_if<__hip_is_normalized_channel_type>::value, bool>::type> +{ + using type = HIP_vector_type<__hip_tex_ret_t, rank>; +}; template static __forceinline__ __device__ __hip_tex_ret_t tex1Dfetch(texture t, int x) @@ -229,20 +315,48 @@ static __forceinline__ __device__ __hip_tex_ret_t tex3DGrad(texture return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); } -template -struct __hip_tex2dgather_ret {}; +template < + typename T, + hipTextureReadMode readMode, + typename Enable = void> +struct __hip_tex2dgather_ret +{ + static_assert(std::is_same::value, "Invalid channel type!"); +}; -template -using __hip_tex2dgather_ret_t = typename __hip_tex2dgather_ret::type; +template < + typename T, + hipTextureReadMode readMode> +using __hip_tex2dgather_ret_t = typename __hip_tex2dgather_ret::type; template -struct __hip_tex2dgather_ret { using type = HIP_vector_type; }; +struct __hip_tex2dgather_ret< + T, + hipReadModeElementType, + typename std::enable_if<__hip_is_channel_type::value, bool>::type> +{ + using type = HIP_vector_type; +}; -template -struct __hip_tex2dgather_ret, hipReadModeElementType> { using type = HIP_vector_type; }; +template< + typename T, + unsigned int rank> +struct __hip_tex2dgather_ret< + HIP_vector_type, + hipReadModeElementType, + typename std::enable_if<__hip_is_channel_type>::value, bool>::type> +{ + using type = HIP_vector_type; +}; template -struct __hip_tex2dgather_ret { using type = float4; }; +struct __hip_tex2dgather_ret< + T, + hipReadModeNormalizedFloat, + typename std::enable_if<__hip_is_normalized_channel_type::value, bool>::type> +{ + using type = float4; +}; template static __forceinline__ __device__ __hip_tex2dgather_ret_t tex2Dgather(texture t, float x, float y, int comp=0) From 117f0ab1020eb7ea14c6366ce528d7eaed5166b0 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 13:09:33 -0400 Subject: [PATCH 104/154] Add constraints to texture indirect functions Similar to the previous patch, this change adds type constraints to texture indirect functions. Since we don't have to deduce the return type for these, we simply just have to check if the user provided a valid channel type. Change-Id: Ia094bd6126e01df2ea90902c9aa59cb6cfe85773 --- .../hip/hcc_detail/texture_fetch_functions.h | 26 +-- .../hcc_detail/texture_indirect_functions.h | 211 ++++++++++++++---- 2 files changed, 178 insertions(+), 59 deletions(-) diff --git a/include/hip/hcc_detail/texture_fetch_functions.h b/include/hip/hcc_detail/texture_fetch_functions.h index 39ac3ecd89..23a850c52c 100644 --- a/include/hip/hcc_detail/texture_fetch_functions.h +++ b/include/hip/hcc_detail/texture_fetch_functions.h @@ -35,7 +35,7 @@ THE SOFTWARE. unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; template -struct __hip_is_channel_type +struct __hip_is_tex_channel_type { static constexpr bool value = std::is_same::value || @@ -50,17 +50,17 @@ struct __hip_is_channel_type template< typename T, unsigned int rank> -struct __hip_is_channel_type> +struct __hip_is_tex_channel_type> { static constexpr bool value = - __hip_is_channel_type::value && + __hip_is_tex_channel_type::value && ((rank == 1) || (rank == 2) || (rank == 4)); }; template -struct __hip_is_normalized_channel_type +struct __hip_is_tex_normalized_channel_type { static constexpr bool value = std::is_same::value || @@ -72,10 +72,10 @@ struct __hip_is_normalized_channel_type template< typename T, unsigned int rank> -struct __hip_is_normalized_channel_type> +struct __hip_is_tex_normalized_channel_type> { static constexpr bool value = - __hip_is_normalized_channel_type::value && + __hip_is_tex_normalized_channel_type::value && ((rank == 1) || (rank == 2) || (rank == 4)); @@ -99,7 +99,7 @@ template struct __hip_tex_ret< T, hipReadModeElementType, - typename std::enable_if<__hip_is_channel_type::value, bool>::type> + typename std::enable_if<__hip_is_tex_channel_type::value, bool>::type> { using type = T; }; @@ -110,7 +110,7 @@ template< struct __hip_tex_ret< HIP_vector_type, hipReadModeElementType, - typename std::enable_if<__hip_is_channel_type>::value, bool>::type> + typename std::enable_if<__hip_is_tex_channel_type>::value, bool>::type> { using type = HIP_vector_type<__hip_tex_ret_t, rank>; }; @@ -119,7 +119,7 @@ template struct __hip_tex_ret< T, hipReadModeNormalizedFloat, - typename std::enable_if<__hip_is_normalized_channel_type::value, bool>::type> + typename std::enable_if<__hip_is_tex_normalized_channel_type::value, bool>::type> { using type = float; }; @@ -130,7 +130,7 @@ template< struct __hip_tex_ret< HIP_vector_type, hipReadModeNormalizedFloat, - typename std::enable_if<__hip_is_normalized_channel_type>::value, bool>::type> + typename std::enable_if<__hip_is_tex_normalized_channel_type>::value, bool>::type> { using type = HIP_vector_type<__hip_tex_ret_t, rank>; }; @@ -333,7 +333,7 @@ template struct __hip_tex2dgather_ret< T, hipReadModeElementType, - typename std::enable_if<__hip_is_channel_type::value, bool>::type> + typename std::enable_if<__hip_is_tex_channel_type::value, bool>::type> { using type = HIP_vector_type; }; @@ -344,7 +344,7 @@ template< struct __hip_tex2dgather_ret< HIP_vector_type, hipReadModeElementType, - typename std::enable_if<__hip_is_channel_type>::value, bool>::type> + typename std::enable_if<__hip_is_tex_channel_type>::value, bool>::type> { using type = HIP_vector_type; }; @@ -353,7 +353,7 @@ template struct __hip_tex2dgather_ret< T, hipReadModeNormalizedFloat, - typename std::enable_if<__hip_is_normalized_channel_type::value, bool>::type> + typename std::enable_if<__hip_is_tex_normalized_channel_type::value, bool>::type> { using type = float4; }; diff --git a/include/hip/hcc_detail/texture_indirect_functions.h b/include/hip/hcc_detail/texture_indirect_functions.h index 56784f433a..4facce436e 100644 --- a/include/hip/hcc_detail/texture_indirect_functions.h +++ b/include/hip/hcc_detail/texture_indirect_functions.h @@ -28,11 +28,40 @@ THE SOFTWARE. #include #include +#include + #define TEXTURE_OBJECT_PARAMETERS_INIT \ unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)textureObject; \ unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD; -template +template +struct __hip_is_itex_channel_type +{ + static constexpr bool value = + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value; +}; + +template< + typename T, + unsigned int rank> +struct __hip_is_itex_channel_type> +{ + static constexpr bool value = + __hip_is_itex_channel_type::value && + ((rank == 1) || + (rank == 2) || + (rank == 4)); +}; + +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex1Dfetch(hipTextureObject_t textureObject, int x) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -40,13 +69,17 @@ static __device__ T tex1Dfetch(hipTextureObject_t textureObject, int x) return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex1Dfetch(T *ptr, hipTextureObject_t textureObject, int x) { *ptr = tex1Dfetch(textureObject, x); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex1D(hipTextureObject_t textureObject, float x) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -54,13 +87,17 @@ static __device__ T tex1D(hipTextureObject_t textureObject, float x) return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex1D(T *ptr, hipTextureObject_t textureObject, float x) { *ptr = tex1D(textureObject, x); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex2D(hipTextureObject_t textureObject, float x, float y) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -68,13 +105,17 @@ static __device__ T tex2D(hipTextureObject_t textureObject, float x, float y) return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex2D(T *ptr, hipTextureObject_t textureObject, float x, float y) { *ptr = tex2D(textureObject, x, y); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex3D(hipTextureObject_t textureObject, float x, float y, float z) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -82,13 +123,17 @@ static __device__ T tex3D(hipTextureObject_t textureObject, float x, float y, fl return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex3D(T *ptr, hipTextureObject_t textureObject, float x, float y, float z) { *ptr = tex3D(textureObject, x, y, z); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex1DLayered(hipTextureObject_t textureObject, float x, int layer) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -96,13 +141,17 @@ static __device__ T tex1DLayered(hipTextureObject_t textureObject, float x, int return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex1DLayered(T *ptr, hipTextureObject_t textureObject, float x, int layer) { *ptr = tex1DLayered(textureObject, x, layer); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex2DLayered(hipTextureObject_t textureObject, float x, float y, int layer) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -110,13 +159,17 @@ static __device__ T tex2DLayered(hipTextureObject_t textureObject, float x, floa return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex2DLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer) { *ptr = tex1DLayered(textureObject, x, y, layer); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T texCubemap(hipTextureObject_t textureObject, float x, float y, float z) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -124,13 +177,17 @@ static __device__ T texCubemap(hipTextureObject_t textureObject, float x, float return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void texCubemap(T *ptr, hipTextureObject_t textureObject, float x, float y, float z) { *ptr = texCubemap(textureObject, x, y, z); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T texCubemapLayered(hipTextureObject_t textureObject, float x, float y, float z, int layer) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -138,13 +195,17 @@ static __device__ T texCubemapLayered(hipTextureObject_t textureObject, float x, return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void texCubemapLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer) { *ptr = texCubemapLayered(textureObject, x, y, z, layer); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex2Dgather(hipTextureObject_t textureObject, float x, float y, int comp = 0) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -173,13 +234,17 @@ static __device__ T tex2Dgather(hipTextureObject_t textureObject, float x, float return {}; } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex2Dgather(T *ptr, hipTextureObject_t textureObject, float x, float y, int comp = 0) { *ptr = texCubemapLayered(textureObject, x, y, comp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex1DLod(hipTextureObject_t textureObject, float x, float level) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -187,13 +252,17 @@ static __device__ T tex1DLod(hipTextureObject_t textureObject, float x, float le return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex1DLod(T *ptr, hipTextureObject_t textureObject, float x, float level) { *ptr = tex1DLod(textureObject, x, level); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex2DLod(hipTextureObject_t textureObject, float x, float y, float level) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -201,13 +270,17 @@ static __device__ T tex2DLod(hipTextureObject_t textureObject, float x, float y, return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex2DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float level) { *ptr = tex2DLod(textureObject, x, y, level); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex3DLod(hipTextureObject_t textureObject, float x, float y, float z, float level) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -215,13 +288,17 @@ static __device__ T tex3DLod(hipTextureObject_t textureObject, float x, float y, return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex3DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level) { *ptr = tex3DLod(textureObject, x, y, z, level); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex1DLayeredLod(hipTextureObject_t textureObject, float x, int layer, float level) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -229,13 +306,17 @@ static __device__ T tex1DLayeredLod(hipTextureObject_t textureObject, float x, i return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex1DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, int layer, float level) { *ptr = tex1DLayeredLod(textureObject, x, layer, level); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex2DLayeredLod(hipTextureObject_t textureObject, float x, float y, int layer, float level) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -243,13 +324,17 @@ static __device__ T tex2DLayeredLod(hipTextureObject_t textureObject, float x, return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex2DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float level) { *ptr = tex2DLayeredLod(textureObject, x, y, layer, level); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T texCubemapLod(hipTextureObject_t textureObject, float x, float y, float z, float level) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -257,13 +342,17 @@ static __device__ T texCubemapLod(hipTextureObject_t textureObject, float x, flo return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void texCubemapLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level) { *ptr = texCubemapLod(textureObject, x, y, z, level); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T texCubemapGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -273,13 +362,17 @@ static __device__ T texCubemapGrad(hipTextureObject_t textureObject, float x, fl return {}; } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void texCubemapGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) { *ptr = texCubemapGrad(textureObject, x, y, z, dPdx, dPdy); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T texCubemapLayeredLod(hipTextureObject_t textureObject, float x, float y, float z, int layer, float level) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -287,13 +380,17 @@ static __device__ T texCubemapLayeredLod(hipTextureObject_t textureObject, float return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void texCubemapLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float level) { *ptr = texCubemapLayeredLod(textureObject, x, y, z, layer, level); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex1DGrad(hipTextureObject_t textureObject, float x, float dPdx, float dPdy) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -301,13 +398,17 @@ static __device__ T tex1DGrad(hipTextureObject_t textureObject, float x, float d return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex1DGrad(T *ptr, hipTextureObject_t textureObject, float x, float dPdx, float dPdy) { *ptr = tex1DGrad(textureObject, x, dPdx, dPdy); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex2DGrad(hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -315,13 +416,17 @@ static __device__ T tex2DGrad(hipTextureObject_t textureObject, float x, float y return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex2DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy) { *ptr = tex2DGrad(textureObject, x, y, dPdx, dPdy); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex3DGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -329,13 +434,17 @@ static __device__ T tex3DGrad(hipTextureObject_t textureObject, float x, float y return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex3DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy) { *ptr = tex3DGrad(textureObject, x, y, z, dPdx, dPdy); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex1DLayeredGrad(hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -343,13 +452,17 @@ static __device__ T tex1DLayeredGrad(hipTextureObject_t textureObject, float x, return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex1DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy) { *ptr = tex1DLayeredGrad(textureObject, x, layer, dPdx, dPdy); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T tex2DLayeredGrad(hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -357,13 +470,17 @@ static __device__ T tex2DLayeredGrad(hipTextureObject_t textureObject, float x, return *reinterpret_cast(&tmp); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void tex2DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy) { *ptr = tex2DLayeredGrad(textureObject, x, y, layer, dPdx, dPdy); } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ T texCubemapLayeredGrad(hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy) { TEXTURE_OBJECT_PARAMETERS_INIT @@ -373,7 +490,9 @@ static __device__ T texCubemapLayeredGrad(hipTextureObject_t textureObject, flo return {}; } -template +template < + typename T, + typename std::enable_if<__hip_is_itex_channel_type::value>::type* = nullptr> static __device__ void texCubemapLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy) { *ptr = texCubemapLayeredGrad(textureObject, x, y, z, layer, dPdx, dPdy); From 292d008a6467ab9e7dd3ffec90aa7195b2e89113 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Wed, 18 Mar 2020 12:23:11 -0400 Subject: [PATCH 105/154] Start the lifetime of the texture reference reinterpret_cast<> doesn't create an object, so the texref is actually unitiliazed. This may lead to garbage data in some of its struct members. Initialize it by performing a placement new. The constructer should set all of its members to default values. There's no way currently to extract the channel type, so use single channel char for now. Change-Id: I41b305a75bb3f30130324de785099f55b3e130c7 --- vdi/hip_module.cpp | 4 ++++ vdi/hip_platform.cpp | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/vdi/hip_module.cpp b/vdi/hip_module.cpp index 66955fcf3d..b92155e518 100644 --- a/vdi/hip_module.cpp +++ b/vdi/hip_module.cpp @@ -528,6 +528,10 @@ hipError_t hipModuleGetTexRef(textureReference** texRef, hipModule_t hmod, const HIP_RETURN(hipErrorNotFound); } + // Texture references created by HIP driver API + // have the default read mode set to normalized float. + (*texRef)->readMode = hipReadModeNormalizedFloat; + HIP_RETURN(hipSuccess); } diff --git a/vdi/hip_platform.cpp b/vdi/hip_platform.cpp index 57a37b02a5..4b0db00b1a 100644 --- a/vdi/hip_platform.cpp +++ b/vdi/hip_platform.cpp @@ -361,7 +361,8 @@ bool PlatformState::getTexRef(const char* hostVar, hipModule_t hmod, textureRefe return false; } - *texRef = reinterpret_cast(dvar->shadowVptr); + *texRef = new (dvar->shadowVptr) texture{}; + return true; } From 99e744ab4a058614b9ff2930991d7437ec962345 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 14:10:06 -0400 Subject: [PATCH 106/154] Don't hardcode the texture read mode The readmode needs to be inferred from the template arguments. Change-Id: I067037035e2492a24eac47e16d4015f879be0ea7 --- include/hip/hcc_detail/hip_texture_types.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/hip/hcc_detail/hip_texture_types.h b/include/hip/hcc_detail/hip_texture_types.h index fcd6d69dbe..e82ef30f00 100644 --- a/include/hip/hcc_detail/hip_texture_types.h +++ b/include/hip/hcc_detail/hip_texture_types.h @@ -57,7 +57,7 @@ struct __HIP_TEXTURE_ATTRIB texture : public textureReference { texture(int norm = 0, enum hipTextureFilterMode fMode = hipFilterModePoint, enum hipTextureAddressMode aMode = hipAddressModeClamp) { normalized = norm; - readMode = hipReadModeNormalizedFloat; + readMode = mode; filterMode = fMode; addressMode[0] = aMode; addressMode[1] = aMode; @@ -69,7 +69,7 @@ struct __HIP_TEXTURE_ATTRIB texture : public textureReference { texture(int norm, enum hipTextureFilterMode fMode, enum hipTextureAddressMode aMode, struct hipChannelFormatDesc desc) { normalized = norm; - readMode = hipReadModeNormalizedFloat; + readMode = mode; filterMode = fMode; addressMode[0] = aMode; addressMode[1] = aMode; From 551bcc6293ac397265d6950c65b8ea9ad8d34354 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 16:44:11 -0400 Subject: [PATCH 107/154] Add missing mipmap API entries Introduce hipFreeMipmappedArray(), hipMallocMipmappedArray() and hipGetMipmappedArrayLevel() APIs. Change-Id: I878228c79fa1c54536c17d6baf45f83d51d2b1c7 --- include/hip/hcc_detail/hip_runtime_api.h | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index df27b851c4..85b8e11eae 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -1939,6 +1939,15 @@ hipError_t hipMalloc3D(hipPitchedPtr* pitchedDevPtr, hipExtent extent); */ hipError_t hipFreeArray(hipArray* array); +/** + * @brief Frees a mipmapped array on the device + * + * @param[in] mipmappedArray - Pointer to mipmapped array to free + * + * @return #hipSuccess, #hipErrorInvalidValue + */ +hipError_t hipFreeMipmappedArray(hipMipmappedArray_t mipmappedArray); + /** * @brief Allocate an array on the device. * @@ -1953,6 +1962,39 @@ hipError_t hipFreeArray(hipArray* array); hipError_t hipMalloc3DArray(hipArray** array, const struct hipChannelFormatDesc* desc, struct hipExtent extent, unsigned int flags); + +/** + * @brief Allocate a mipmapped array on the device + * + * @param[out] mipmappedArray - Pointer to allocated mipmapped array in device memory + * @param[in] desc - Requested channel format + * @param[in] extent - Requested allocation size (width field in elements) + * @param[in] numLevels - Number of mipmap levels to allocate + * @param[in] flags - Flags for extensions + * + * @return #hipSuccess, #hipErrorInvalidValue, #hipErrorMemoryAllocation + */ +hipError_t hipMallocMipmappedArray( + hipMipmappedArray_t *mipmappedArray, + const struct hipChannelFormatDesc* desc, + struct hipExtent extent, + unsigned int numLevels, + unsigned int flags __dparm(0)); + +/** + * @brief Gets a mipmap level of a HIP mipmapped array + * + * @param[out] levelArray - Returned mipmap level HIP array + * @param[in] mipmappedArray - HIP mipmapped array + * @param[in] level - Mipmap level + * + * @return #hipSuccess, #hipErrorInvalidValue + */ +hipError_t hipGetMipmappedArrayLevel( + hipArray_t *levelArray, + hipMipmappedArray_const_t mipmappedArray, + unsigned int level); + /** * @brief Copies data between host and device. * From 7190fa518e31fcf8f12f1a9bd066fd06857a78fb Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 16:47:54 -0400 Subject: [PATCH 108/154] Correct the declaration of hipBindTexture() The texture reference needs to be passed as a constant pointer. Change-Id: I36ca0bddaba30becfc2ce70dd9e5b7db66c57f27 --- include/hip/hcc_detail/hip_runtime_api.h | 9 +++++++++ vdi/hip_texture.cpp | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 85b8e11eae..232ef44e7f 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -3362,8 +3362,17 @@ const char* hipKernelNameRef(const hipFunction_t f); class TlsData; +#if __HIP_VDI__ +hipError_t hipBindTexture( + size_t* offset, + const textureReference* tex, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t size = UINT_MAX); +#else hipError_t hipBindTexture(size_t* offset, textureReference* tex, const void* devPtr, const hipChannelFormatDesc* desc, size_t size = UINT_MAX); +#endif hipError_t ihipBindTextureImpl(TlsData *tls, int dim, enum hipTextureReadMode readMode, size_t* offset, const void* devPtr, const struct hipChannelFormatDesc* desc, diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index db405810a6..68c45308e5 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -607,7 +607,7 @@ hipError_t hipUnbindTexture(const textureReference* texref) { } hipError_t hipBindTexture(size_t* offset, - textureReference* texref, + const textureReference* texref, const void* devPtr, const hipChannelFormatDesc* desc, size_t size) { From 2d773997479f7e4206077bc1999bad390b3972c6 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 16:49:55 -0400 Subject: [PATCH 109/154] Correct the declaration of hipBindTextureToArray() The texture reference needs to be passed as a constant pointer. Change-Id: Iff171626536071fb2020cfff7132ec930577b1b9 --- include/hip/hcc_detail/hip_runtime_api.h | 7 +++++++ vdi/hip_texture.cpp | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 232ef44e7f..3a75fb680f 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -3443,8 +3443,15 @@ hipError_t hipBindTexture2D(size_t* offset, struct texture& te } // C API +#if __HIP_VDI__ +hipError_t hipBindTextureToArray( + const textureReference* tex, + hipArray_const_t array, + const hipChannelFormatDesc* desc); +#else hipError_t hipBindTextureToArray(textureReference* tex, hipArray_const_t array, const hipChannelFormatDesc* desc); +#endif hipError_t ihipBindTextureToArrayImpl(TlsData *tls, int dim, enum hipTextureReadMode readMode, hipArray_const_t array, diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 68c45308e5..0c9189a6ad 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -496,7 +496,7 @@ hipError_t ihipBindTextureToArray(const textureReference* texref, return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, &resViewDesc); } -hipError_t hipBindTextureToArray(textureReference* texref, +hipError_t hipBindTextureToArray(const textureReference* texref, hipArray_const_t array, const hipChannelFormatDesc* desc) { HIP_INIT_API(hipBindTextureToArray, texref, array, desc); From 3e460ab514071bbc81242417fa9cc91554fdc1d3 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 16:53:54 -0400 Subject: [PATCH 110/154] Correct the declaration of hipBindTexture2D() The texture reference needs to be passed as a constant pointer. Change-Id: Idde461f0f328ac87ce677b6bab3203161b514cbf --- include/hip/hcc_detail/hip_runtime_api.h | 11 +++++++++++ vdi/hip_texture.cpp | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 3a75fb680f..fa6b7d95f0 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -3420,9 +3420,20 @@ hipError_t hipBindTexture(size_t* offset, struct texture& tex, } // C API +#if __HIP_VDI__ +hipError_t hipBindTexture2D( + size_t* offset, + const textureReference* tex, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t width, + size_t height, + size_t pitch); +#else hipError_t hipBindTexture2D(size_t* offset, textureReference* tex, const void* devPtr, const hipChannelFormatDesc* desc, size_t width, size_t height, size_t pitch); +#endif hipError_t ihipBindTexture2DImpl(int dim, enum hipTextureReadMode readMode, size_t* offset, const void* devPtr, const struct hipChannelFormatDesc* desc, diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 0c9189a6ad..7aa5204424 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -457,7 +457,7 @@ hipError_t ihipBindTexture2D(size_t* offset, } hipError_t hipBindTexture2D(size_t* offset, - textureReference* texref, + const textureReference* texref, const void* devPtr, const hipChannelFormatDesc* desc, size_t width, From c7407a3b57dad97aa568743f84fd5c0edba9f312 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 17:22:17 -0400 Subject: [PATCH 111/154] Correct the definition of ... hipBindTextureToMipmappedArray() The texture reference needs to be passed as a constant pointer. Change-Id: I6d31204c7f2325a5bc1e8b6e089fd9f8d21d1d78 --- vdi/hip_texture.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 7aa5204424..2e4e118687 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -534,7 +534,7 @@ hipError_t ihipBindTextureToArrayImpl(TlsData *tls, HIP_RETURN(ihipBindTextureToArray(texref, array, &desc, readMode)); } -hipError_t ihipBindTextureToMipmappedArray(textureReference* texref, +hipError_t ihipBindTextureToMipmappedArray(const textureReference* texref, hipMipmappedArray_const_t mipmappedArray, const hipChannelFormatDesc* desc, hipTextureReadMode readMode) { @@ -561,7 +561,7 @@ hipError_t ihipBindTextureToMipmappedArray(textureReference* texref, return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, &resViewDesc); } -hipError_t hipBindTextureToMipmappedArray(textureReference* texref, +hipError_t hipBindTextureToMipmappedArray(const textureReference* texref, hipMipmappedArray_const_t mipmappedArray, const hipChannelFormatDesc* desc) { HIP_INIT_API(hipBindTextureToMipmappedArray, texref, mipmappedArray, desc); From 7385a032ae3efe82fe4ffc1354e4b9e3448abf7f Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 17:38:07 -0400 Subject: [PATCH 112/154] Correctly infer the texture read mode Currently we extract the read mode from the ihip*impl() calls, which is not correct. We should be getting it from the texture itself directly. Change-Id: Idf6449fefa395a887138a252e8ea937a6897e600 --- vdi/hip_texture.cpp | 59 ++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 2e4e118687..81cf1bcd77 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -386,8 +386,7 @@ hipError_t ihipBindTexture(size_t* offset, const textureReference* texref, const void* devPtr, const hipChannelFormatDesc* desc, - size_t size, - hipTextureReadMode readMode) { + size_t size) { if ((texref == nullptr) || (devPtr == nullptr) || (desc == nullptr)) { @@ -412,7 +411,7 @@ hipError_t ihipBindTexture(size_t* offset, resDesc.res.linear.desc = *desc; resDesc.res.linear.sizeInBytes = size; - hipTextureDesc texDesc = hip::getTextureDesc(texref, readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texref, texref->readMode); return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); } @@ -423,8 +422,7 @@ hipError_t ihipBindTexture2D(size_t* offset, const hipChannelFormatDesc* desc, size_t width, size_t height, - size_t pitch, - hipTextureReadMode readMode) { + size_t pitch) { if ((texref == nullptr) || (devPtr == nullptr) || (desc == nullptr)) { @@ -451,7 +449,7 @@ hipError_t ihipBindTexture2D(size_t* offset, resDesc.res.pitch2D.height = height; resDesc.res.pitch2D.pitchInBytes = pitch; - hipTextureDesc texDesc = hip::getTextureDesc(texref, readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texref, texref->readMode); return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); } @@ -465,14 +463,12 @@ hipError_t hipBindTexture2D(size_t* offset, size_t pitch) { HIP_INIT_API(hipBindTexture2D, offset, texref, devPtr, desc, width, height, pitch); - // TODO need compiler support to extract the read mode from textureReference. - HIP_RETURN(ihipBindTexture2D(offset, texref, devPtr, desc, width, height, pitch, hipReadModeElementType)); + HIP_RETURN(ihipBindTexture2D(offset, texref, devPtr, desc, width, height, pitch)); } hipError_t ihipBindTextureToArray(const textureReference* texref, hipArray_const_t array, - const hipChannelFormatDesc* desc, - hipTextureReadMode readMode) { + const hipChannelFormatDesc* desc) { if ((texref == nullptr) || (array == nullptr) || (desc == nullptr)) { @@ -488,7 +484,7 @@ hipError_t ihipBindTextureToArray(const textureReference* texref, resDesc.resType = hipResourceTypeArray; resDesc.res.array.array = const_cast(array); - hipTextureDesc texDesc = hip::getTextureDesc(texref, readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texref, texref->readMode); hipResourceViewFormat format = hip::getResourceViewFormat(*desc); hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(array, format); @@ -501,8 +497,7 @@ hipError_t hipBindTextureToArray(const textureReference* texref, const hipChannelFormatDesc* desc) { HIP_INIT_API(hipBindTextureToArray, texref, array, desc); - // TODO need compiler support to extract the read mode from textureReference. - HIP_RETURN(ihipBindTextureToArray(texref, array, desc, hipReadModeElementType)); + HIP_RETURN(ihipBindTextureToArray(texref, array, desc)); } hipError_t ihipBindTextureImpl(TlsData *tls, @@ -515,9 +510,11 @@ hipError_t ihipBindTextureImpl(TlsData *tls, textureReference* texref) { HIP_INIT_API(ihipBindTextureImpl, tls, dim, readMode, offset, devPtr, desc, size, texref); - (void)dim; // Silence compiler warnings. + // Silence compiler warnings. + (void)dim; + (void)readMode; - HIP_RETURN(ihipBindTexture(offset, texref, devPtr, desc, size, readMode)); + HIP_RETURN(ihipBindTexture(offset, texref, devPtr, desc, size)); } hipError_t ihipBindTextureToArrayImpl(TlsData *tls, @@ -529,15 +526,16 @@ hipError_t ihipBindTextureToArrayImpl(TlsData *tls, // TODO overload operator<<(ostream&, hipChannelFormatDesc&). HIP_INIT_API(ihipBindTextureToArrayImpl, tls, dim, readMode, array, &desc, texref); - (void)dim; // Silence compiler warnings. + // Silence compiler warnings. + (void)dim; + (void)readMode; - HIP_RETURN(ihipBindTextureToArray(texref, array, &desc, readMode)); + HIP_RETURN(ihipBindTextureToArray(texref, array, &desc)); } hipError_t ihipBindTextureToMipmappedArray(const textureReference* texref, hipMipmappedArray_const_t mipmappedArray, - const hipChannelFormatDesc* desc, - hipTextureReadMode readMode) { + const hipChannelFormatDesc* desc) { if ((texref == nullptr) || (mipmappedArray == nullptr) || (desc == nullptr)) { @@ -553,7 +551,7 @@ hipError_t ihipBindTextureToMipmappedArray(const textureReference* texref, resDesc.resType = hipResourceTypeMipmappedArray; resDesc.res.mipmap.mipmap = const_cast(mipmappedArray); - hipTextureDesc texDesc = hip::getTextureDesc(texref, readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texref, texref->readMode); hipResourceViewFormat format = hip::getResourceViewFormat(*desc); hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(mipmappedArray, format); @@ -566,8 +564,7 @@ hipError_t hipBindTextureToMipmappedArray(const textureReference* texref, const hipChannelFormatDesc* desc) { HIP_INIT_API(hipBindTextureToMipmappedArray, texref, mipmappedArray, desc); - // TODO need compiler support to extract the read mode from textureReference. - HIP_RETURN(ihipBindTextureToMipmappedArray(texref, mipmappedArray, desc, hipReadModeElementType)); + HIP_RETURN(ihipBindTextureToMipmappedArray(texref, mipmappedArray, desc)); } hipError_t ihipBindTexture2DImpl(int dim, @@ -581,9 +578,11 @@ hipError_t ihipBindTexture2DImpl(int dim, size_t pitch) { HIP_INIT_API(ihipBindTexture2DImpl, dim, readMode, offset, devPtr, desc, width, height, texref, pitch); - (void)dim; // Silence compiler warnings. + // Silence compiler warnings. + (void)dim; + (void)readMode; - HIP_RETURN(ihipBindTexture2D(offset, texref, devPtr, desc, width, height, pitch, readMode)); + HIP_RETURN(ihipBindTexture2D(offset, texref, devPtr, desc, width, height, pitch)); } hipError_t ihipUnbindTextureImpl(const hipTextureObject_t& textureObject) { @@ -613,8 +612,7 @@ hipError_t hipBindTexture(size_t* offset, size_t size) { HIP_INIT_API(hipBindTexture, offset, texref, devPtr, desc, size); - // TODO need compiler support to extract the read mode from textureReference. - HIP_RETURN(ihipBindTexture(offset, texref, devPtr, desc, size, hipReadModeElementType)); + HIP_RETURN(ihipBindTexture(offset, texref, devPtr, desc, size)); } hipError_t hipGetChannelDesc(hipChannelFormatDesc* desc, @@ -791,8 +789,7 @@ hipError_t hipTexRefSetArray(textureReference* texRef, resDesc.resType = hipResourceTypeArray; resDesc.res.array.array = const_cast(array); - // TODO need compiler support to extract the read mode from textureReference. - hipTextureDesc texDesc = hip::getTextureDesc(texRef, hipReadModeElementType); + hipTextureDesc texDesc = hip::getTextureDesc(texRef, texRef->readMode); hipResourceViewFormat format = hip::getResourceViewFormat(hip::getChannelFormatDesc(texRef->numChannels, texRef->format)); hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(array, format); @@ -923,9 +920,11 @@ hipError_t ihipBindTextureToMipmappedArrayImpl(TlsData *tls, // TODO overload operator<<(ostream&, hipChannelFormatDesc&). HIP_INIT_API(ihipBindTextureToMipmappedArrayImpl, tls, dim, readMode, mipmappedArray, &desc, texref); - (void)dim; // Silence compiler warnings. + // Silence compiler warnings. + (void)dim; + (void)readMode; - HIP_RETURN(ihipBindTextureToMipmappedArray(texref, mipmappedArray, &desc, readMode)); + HIP_RETURN(ihipBindTextureToMipmappedArray(texref, mipmappedArray, &desc)); } hipError_t hipTexRefGetBorderColor(float* pBorderColor, From 5429b40afeac311a48df31adb01b2dc37d3b11fd Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 17:43:34 -0400 Subject: [PATCH 113/154] Rework the texture C++ API Currently the texture C++ API is forwarded to the ihip*Impl() calls, which are not even a part of Cuda. These should be forwarded to their respective Cuda C APIs instead. This change also fixes a bug with hipUnbindTexture() creating a dangling pointer. Change-Id: Ifafc9d106855a11bec84a18ea214b3d89e39990d --- include/hip/hcc_detail/hip_runtime_api.h | 126 +++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index fa6b7d95f0..2a781154e1 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -3374,9 +3374,11 @@ hipError_t hipBindTexture(size_t* offset, textureReference* tex, const void* dev const hipChannelFormatDesc* desc, size_t size = UINT_MAX); #endif +#if !__HIP_VDI__ hipError_t ihipBindTextureImpl(TlsData *tls, int dim, enum hipTextureReadMode readMode, size_t* offset, const void* devPtr, const struct hipChannelFormatDesc* desc, size_t size, textureReference* tex); +#endif /* * @brief hipBindTexture Binds size bytes of the memory area pointed to by @p devPtr to the texture @@ -3393,11 +3395,13 @@ hipError_t ihipBindTextureImpl(TlsData *tls, int dim, enum hipTextureReadMode re * @param[in] size - Size of the memory area pointed to by devPtr * @return #hipSuccess, #hipErrorInvalidValue, #hipErrorMemoryFree, #hipErrorUnknown **/ +#if !__HIP_VDI__ template hipError_t hipBindTexture(size_t* offset, struct texture& tex, const void* devPtr, const struct hipChannelFormatDesc& desc, size_t size = UINT_MAX) { return ihipBindTextureImpl(nullptr, dim, readMode, offset, devPtr, &desc, size, &tex); } +#endif /* * @brief hipBindTexture Binds size bytes of the memory area pointed to by @p devPtr to the texture @@ -3413,11 +3417,13 @@ hipError_t hipBindTexture(size_t* offset, struct texture& tex, * @param[in] size - Size of the memory area pointed to by devPtr * @return #hipSuccess, #hipErrorInvalidValue, #hipErrorMemoryFree, #hipErrorUnknown **/ +#if !__HIP_VDI__ template hipError_t hipBindTexture(size_t* offset, struct texture& tex, const void* devPtr, size_t size = UINT_MAX) { return ihipBindTextureImpl(nullptr, dim, readMode, offset, devPtr, &(tex.channelDesc), size, &tex); } +#endif // C API #if __HIP_VDI__ @@ -3435,23 +3441,29 @@ hipError_t hipBindTexture2D(size_t* offset, textureReference* tex, const void* d size_t pitch); #endif +#if !__HIP_VDI__ hipError_t ihipBindTexture2DImpl(int dim, enum hipTextureReadMode readMode, size_t* offset, const void* devPtr, const struct hipChannelFormatDesc* desc, size_t width, size_t height, textureReference* tex, size_t pitch); +#endif +#if !__HIP_VDI__ template hipError_t hipBindTexture2D(size_t* offset, struct texture& tex, const void* devPtr, size_t width, size_t height, size_t pitch) { return ihipBindTexture2DImpl(dim, readMode, offset, devPtr, &(tex.channelDesc), width, height, &tex); } +#endif +#if !__HIP_VDI__ template hipError_t hipBindTexture2D(size_t* offset, struct texture& tex, const void* devPtr, const struct hipChannelFormatDesc& desc, size_t width, size_t height, size_t pitch) { return ihipBindTexture2DImpl(dim, readMode, offset, devPtr, &desc, width, height, &tex); } +#endif // C API #if __HIP_VDI__ @@ -3464,46 +3476,58 @@ hipError_t hipBindTextureToArray(textureReference* tex, hipArray_const_t array, const hipChannelFormatDesc* desc); #endif +#if !__HIP_VDI__ hipError_t ihipBindTextureToArrayImpl(TlsData *tls, int dim, enum hipTextureReadMode readMode, hipArray_const_t array, const struct hipChannelFormatDesc& desc, textureReference* tex); +#endif +#if !__HIP_VDI__ template hipError_t hipBindTextureToArray(struct texture& tex, hipArray_const_t array) { return ihipBindTextureToArrayImpl(nullptr, dim, readMode, array, tex.channelDesc, &tex); } +#endif +#if !__HIP_VDI__ template hipError_t hipBindTextureToArray(struct texture& tex, hipArray_const_t array, const struct hipChannelFormatDesc& desc) { return ihipBindTextureToArrayImpl(nullptr, dim, readMode, array, desc, &tex); } +#endif +#if !__HIP_VDI__ template inline static hipError_t hipBindTextureToArray(struct texture *tex, hipArray_const_t array, const struct hipChannelFormatDesc* desc) { return ihipBindTextureToArrayImpl(nullptr, dim, readMode, array, *desc, tex); } +#endif // C API hipError_t hipBindTextureToMipmappedArray(const textureReference* tex, hipMipmappedArray_const_t mipmappedArray, const hipChannelFormatDesc* desc); +#if !__HIP_VDI__ template hipError_t hipBindTextureToMipmappedArray(const texture& tex, hipMipmappedArray_const_t mipmappedArray) { return hipSuccess; } +#endif +#if !__HIP_VDI__ template hipError_t hipBindTextureToMipmappedArray(const texture& tex, hipMipmappedArray_const_t mipmappedArray, const hipChannelFormatDesc& desc) { return hipSuccess; } +#endif #if __HIP_VDI__ && !defined(__HCC__) @@ -3544,12 +3568,16 @@ inline hipError_t hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchPara **/ hipError_t hipUnbindTexture(const textureReference* tex); +#if !__HIP_VDI__ extern hipError_t ihipUnbindTextureImpl(const hipTextureObject_t& textureObject); +#endif +#if !__HIP_VDI__ template hipError_t hipUnbindTexture(struct texture& tex) { return ihipUnbindTextureImpl(tex.textureObject); } +#endif hipError_t hipGetChannelDesc(hipChannelFormatDesc* desc, hipArray_const_t array); hipError_t hipGetTextureAlignmentOffset(size_t* offset, const textureReference* texref); @@ -3593,6 +3621,104 @@ hipError_t hipCreateSurfaceObject(hipSurfaceObject_t* pSurfObject, const hipReso hipError_t hipDestroySurfaceObject(hipSurfaceObject_t surfaceObject); +#if __HIP_VDI__ +template +static inline hipError_t hipBindTexture( + size_t *offset, + const struct texture &tex, + const void *devPtr, + size_t size = UINT_MAX) +{ + return hipBindTexture(offset, tex, devPtr, tex.channelDesc, size); +} + +template +static inline hipError_t hipBindTexture( + size_t *offset, + const struct texture &tex, + const void *devPtr, + const struct hipChannelFormatDesc &desc, + size_t size = UINT_MAX) +{ + return hipBindTexture(offset, &tex, devPtr, &desc, size); +} + +template +static inline hipError_t hipBindTexture2D( + size_t *offset, + const struct texture &tex, + const void *devPtr, + size_t width, + size_t height, + size_t pitch) +{ + return hipBindTexture2D(offset, &tex, devPtr, &tex.channelDesc, width, height, pitch); +} + +template +static inline hipError_t hipBindTexture2D( + size_t *offset, + const struct texture &tex, + const void *devPtr, + const struct hipChannelFormatDesc &desc, + size_t width, + size_t height, + size_t pitch) +{ + return hipBindTexture2D(offset, &tex, devPtr, &desc, width, height, pitch); +} + +template +static inline hipError_t hipBindTextureToArray( + const struct texture &tex, + hipArray_const_t array) +{ + struct cudaChannelFormatDesc desc; + hipError_t err = hipGetChannelDesc(&desc, array); + return (err == hipSuccess) ? hipBindTextureToArray(tex, array, desc) : err; +} + +template +static inline hipError_t hipBindTextureToArray( + const struct texture &tex, + hipArray_const_t array, + const struct hipChannelFormatDesc &desc) +{ + return hipBindTextureToArray(&tex, array, &desc); +} + +template +static inline hipError_t hipBindTextureToMipmappedArray( + const struct texture &tex, + hipMipmappedArray_const_t mipmappedArray) +{ + struct hipChannelFormatDesc desc; + hipArray_t levelArray; + hipError_t err = hipGetMipmappedArrayLevel(&levelArray, mipmappedArray, 0); + if (err != hipSuccess) { + return err; + } + err = hipGetChannelDesc(&desc, levelArray); + return (err == hipSuccess) ? hipBindTextureToMipmappedArray(tex, mipmappedArray, desc) : err; +} + +template +static inline hipError_t hipBindTextureToMipmappedArray( + const struct texture &tex, + hipMipmappedArray_const_t mipmappedArray, + const struct cudaChannelFormatDesc &desc) +{ + return hipBindTextureToMipmappedArray(&tex, mipmappedArray, &desc); +} + +template +static inline hipError_t hipUnbindTexture( + const struct texture &tex) +{ + return hipUnbindTexture(&tex); +} +#endif + // doxygen end Texture /** * @} From 2bad9e28219b9d941ee899ef20c0dfea1e22e441 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 17 Mar 2020 18:09:39 -0400 Subject: [PATCH 114/154] Purge the use of ihip*impl() texture APIs These are artifacts left from HIP-HCC and now are not needed by HIP-VDI. Change-Id: Ib25a1081fe6146c8a89659395151e9d5bdaf7519 --- vdi/hip_hcc.def.in | 3 -- vdi/hip_hcc.map.in | 3 -- vdi/hip_texture.cpp | 74 --------------------------------------------- 3 files changed, 80 deletions(-) diff --git a/vdi/hip_hcc.def.in b/vdi/hip_hcc.def.in index 68fc2cd13a..57d3a80719 100644 --- a/vdi/hip_hcc.def.in +++ b/vdi/hip_hcc.def.in @@ -183,7 +183,6 @@ hipBindTexture2D hipBindTextureToArray hipBindTextureToMipmappedArray hipUnbindTexture -ihipUnbindTextureImpl hipGetChannelDesc hipGetTextureAlignmentOffset hipGetTextureReference @@ -198,8 +197,6 @@ hipTexRefGetAddress hipTexRefGetAddressMode hipTexRefGetArray hipTexRefSetArray -ihipBindTextureImpl -ihipBindTextureToArrayImpl hipCreateChannelDesc hipProfilerStart hipProfilerStop diff --git a/vdi/hip_hcc.map.in b/vdi/hip_hcc.map.in index aafc81f4be..70109c4943 100644 --- a/vdi/hip_hcc.map.in +++ b/vdi/hip_hcc.map.in @@ -199,7 +199,6 @@ global: hipBindTextureToArray*; hipBindTextureToMipmappedArray*; hipUnbindTexture*; - ihipUnbindTextureImpl*; hipGetChannelDesc*; hipGetTextureAlignmentOffset*; hipGetTextureReference*; @@ -215,8 +214,6 @@ global: hipTexRefGetArray*; hipTexRefSetArray*; hipCreateChannelDesc*; - ihipBindTextureImpl*; - ihipBindTextureToArrayImpl*; hipHccGetAccelerator*; hipHccGetAcceleratorView*; hipCreateSurfaceObject*; diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 81cf1bcd77..0cdb202641 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -500,39 +500,6 @@ hipError_t hipBindTextureToArray(const textureReference* texref, HIP_RETURN(ihipBindTextureToArray(texref, array, desc)); } -hipError_t ihipBindTextureImpl(TlsData *tls, - int dim, - hipTextureReadMode readMode, - size_t* offset, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t size, - textureReference* texref) { - HIP_INIT_API(ihipBindTextureImpl, tls, dim, readMode, offset, devPtr, desc, size, texref); - - // Silence compiler warnings. - (void)dim; - (void)readMode; - - HIP_RETURN(ihipBindTexture(offset, texref, devPtr, desc, size)); -} - -hipError_t ihipBindTextureToArrayImpl(TlsData *tls, - int dim, - hipTextureReadMode readMode, - hipArray_const_t array, - const hipChannelFormatDesc& desc, - textureReference* texref) { - // TODO overload operator<<(ostream&, hipChannelFormatDesc&). - HIP_INIT_API(ihipBindTextureToArrayImpl, tls, dim, readMode, array, &desc, texref); - - // Silence compiler warnings. - (void)dim; - (void)readMode; - - HIP_RETURN(ihipBindTextureToArray(texref, array, &desc)); -} - hipError_t ihipBindTextureToMipmappedArray(const textureReference* texref, hipMipmappedArray_const_t mipmappedArray, const hipChannelFormatDesc* desc) { @@ -567,31 +534,6 @@ hipError_t hipBindTextureToMipmappedArray(const textureReference* texref, HIP_RETURN(ihipBindTextureToMipmappedArray(texref, mipmappedArray, desc)); } -hipError_t ihipBindTexture2DImpl(int dim, - hipTextureReadMode readMode, - size_t* offset, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t width, - size_t height, - textureReference* texref, - size_t pitch) { - HIP_INIT_API(ihipBindTexture2DImpl, dim, readMode, offset, devPtr, desc, width, height, texref, pitch); - - // Silence compiler warnings. - (void)dim; - (void)readMode; - - HIP_RETURN(ihipBindTexture2D(offset, texref, devPtr, desc, width, height, pitch)); -} - -hipError_t ihipUnbindTextureImpl(const hipTextureObject_t& textureObject) { - // TODO overload operator<<(ostream&, hipTextureObject_t&). - HIP_INIT_API(ihipUnbindTextureImpl, &textureObject); - - HIP_RETURN(ihipDestroyTextureObject(textureObject)); -} - hipError_t hipUnbindTexture(const textureReference* texref) { HIP_INIT_API(hipUnbindTexture, texref); @@ -911,22 +853,6 @@ hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannel return {x, y, z, w, f}; } -hipError_t ihipBindTextureToMipmappedArrayImpl(TlsData *tls, - int dim, - hipTextureReadMode readMode, - hipMipmappedArray_const_t mipmappedArray, - const struct hipChannelFormatDesc& desc, - textureReference* texref) { - // TODO overload operator<<(ostream&, hipChannelFormatDesc&). - HIP_INIT_API(ihipBindTextureToMipmappedArrayImpl, tls, dim, readMode, mipmappedArray, &desc, texref); - - // Silence compiler warnings. - (void)dim; - (void)readMode; - - HIP_RETURN(ihipBindTextureToMipmappedArray(texref, mipmappedArray, &desc)); -} - hipError_t hipTexRefGetBorderColor(float* pBorderColor, textureReference texRef) { // TODO overload operator<<(ostream&, textureReference&). From e8fa3b2589fb04a773127dc69f53f91ded8b0fdf Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Wed, 18 Mar 2020 12:24:05 -0400 Subject: [PATCH 115/154] Program texture flags in a better way Not sure what I was thinking when initially implementing this... Change-Id: Ib82f0f5a86683c08823dd4b59c98259d27151822 --- vdi/hip_conversions.hpp | 32 ++----------------- vdi/hip_texture.cpp | 69 +++++++++++++++++++++-------------------- 2 files changed, 38 insertions(+), 63 deletions(-) diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index c1ed166e16..0801c310eb 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -491,39 +491,11 @@ hipResourceViewFormat getResourceViewFormat(const hipChannelFormatDesc& desc) { } inline -hipTextureReadMode getReadMode(unsigned int flags) { - if (flags & HIP_TRSF_READ_AS_INTEGER) { - return hipReadModeElementType; - } else { - return hipReadModeNormalizedFloat; - } -} - -inline -int getNormalizedCoords(unsigned int flags) { - if (flags & HIP_TRSF_NORMALIZED_COORDINATES) { - return 1; - } else { - return 0; - } -} - -inline -int getSRGB(unsigned int flags) { - if (flags & HIP_TRSF_SRGB) { - return 1; - } else { - return 0; - } -} - -inline -hipTextureDesc getTextureDesc(const textureReference* texRef, - const hipTextureReadMode readMode) { +hipTextureDesc getTextureDesc(const textureReference* texRef) { hipTextureDesc texDesc = {}; std::memcpy(texDesc.addressMode, texRef->addressMode, sizeof(texDesc.addressMode)); texDesc.filterMode = texRef->filterMode; - texDesc.readMode = readMode; + texDesc.readMode = texRef->readMode; texDesc.sRGB = texRef->sRGB; texDesc.normalizedCoords = texRef->normalized; texDesc.maxAnisotropy = texRef->maxAnisotropy; diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 0cdb202641..2d4085d928 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -411,7 +411,7 @@ hipError_t ihipBindTexture(size_t* offset, resDesc.res.linear.desc = *desc; resDesc.res.linear.sizeInBytes = size; - hipTextureDesc texDesc = hip::getTextureDesc(texref, texref->readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texref); return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); } @@ -449,7 +449,7 @@ hipError_t ihipBindTexture2D(size_t* offset, resDesc.res.pitch2D.height = height; resDesc.res.pitch2D.pitchInBytes = pitch; - hipTextureDesc texDesc = hip::getTextureDesc(texref, texref->readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texref); return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); } @@ -484,7 +484,7 @@ hipError_t ihipBindTextureToArray(const textureReference* texref, resDesc.resType = hipResourceTypeArray; resDesc.res.array.array = const_cast(array); - hipTextureDesc texDesc = hip::getTextureDesc(texref, texref->readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texref); hipResourceViewFormat format = hip::getResourceViewFormat(*desc); hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(array, format); @@ -518,7 +518,7 @@ hipError_t ihipBindTextureToMipmappedArray(const textureReference* texref, resDesc.resType = hipResourceTypeMipmappedArray; resDesc.res.mipmap.mipmap = const_cast(mipmappedArray); - hipTextureDesc texDesc = hip::getTextureDesc(texref, texref->readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texref); hipResourceViewFormat format = hip::getResourceViewFormat(*desc); hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(mipmappedArray, format); @@ -619,11 +619,21 @@ hipError_t hipTexRefSetFlags(textureReference* texRef, HIP_RETURN(hipErrorInvalidValue); } - // TODO add textureReference::flags. - // Using textureReference::normalized for this purpose is OK for now, - // because calling hipTexRefGetFlags() on a textureRefence after hipBindTexture() is UB - // due to HIP not differentiating between runtime and driver api. - texRef->normalized = Flags; + texRef->readMode = hipReadModeNormalizedFloat; + texRef->normalized = 0; + texRef->sRGB = 0; + + if (Flags & HIP_TRSF_READ_AS_INTEGER) { + texRef->readMode = hipReadModeElementType; + } + + if (Flags & HIP_TRSF_NORMALIZED_COORDINATES) { + texRef->normalized = 1; + } + + if (Flags & HIP_TRSF_SRGB) { + texRef->sRGB = 1; + } HIP_RETURN(hipSuccess); } @@ -731,7 +741,7 @@ hipError_t hipTexRefSetArray(textureReference* texRef, resDesc.resType = hipResourceTypeArray; resDesc.res.array.array = const_cast(array); - hipTextureDesc texDesc = hip::getTextureDesc(texRef, texRef->readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texRef); hipResourceViewFormat format = hip::getResourceViewFormat(hip::getChannelFormatDesc(texRef->numChannels, texRef->format)); hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(array, format); @@ -801,14 +811,7 @@ hipError_t hipTexRefSetAddress(size_t* ByteOffset, resDesc.res.linear.desc = hip::getChannelFormatDesc(texRef->numChannels, texRef->format); resDesc.res.linear.sizeInBytes = bytes; - // TODO add textureReference::flags. - // Using textureReference::normalized for this purpose is OK for now, - // because calling hipTexRefGetFlags() on a textureRefence after hipBindTexture() - // due to HIP not differentiating between runtime and driver api. - hipTextureReadMode readMode = hip::getReadMode(texRef->normalized); - texRef->sRGB = hip::getSRGB(texRef->normalized); - texRef->normalized = hip::getNormalizedCoords(texRef->normalized); - hipTextureDesc texDesc = hip::getTextureDesc(texRef, readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texRef); HIP_RETURN(ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, nullptr)); } @@ -837,14 +840,7 @@ hipError_t hipTexRefSetAddress2D(textureReference* texRef, resDesc.res.pitch2D.height = desc->Height; resDesc.res.pitch2D.pitchInBytes = Pitch; - // TODO add textureReference::flags. - // Using textureReference::normalized for this purpose is OK for now, - // because calling hipTexRefGetFlags() on a textureRefence after hipBindTexture() - // due to HIP not differentiating between runtime and driver api. - hipTextureReadMode readMode = hip::getReadMode(texRef->normalized); - texRef->sRGB = hip::getSRGB(texRef->normalized); - texRef->normalized = hip::getNormalizedCoords(texRef->normalized); - hipTextureDesc texDesc = hip::getTextureDesc(texRef, readMode); + hipTextureDesc texDesc = hip::getTextureDesc(texRef); HIP_RETURN(ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, nullptr)); } @@ -892,11 +888,19 @@ hipError_t hipTexRefGetFlags(unsigned int* pFlags, HIP_RETURN(hipErrorInvalidValue); } - // TODO add textureReference::flags. - // Using textureReference::normalized for this purpose is OK for now, - // because calling hipTexRefGetFlags() on a textureRefence after hipBindTexture() - // due to HIP not differentiating between runtime and driver api. - *pFlags = texRef.normalized; + *pFlags = 0; + + if (texRef.readMode == hipReadModeElementType) { + *pFlags |= HIP_TRSF_READ_AS_INTEGER; + } + + if (texRef.normalized == 1) { + *pFlags |= HIP_TRSF_NORMALIZED_COORDINATES; + } + + if (texRef.sRGB == 1) { + *pFlags |= HIP_TRSF_SRGB; + } HIP_RETURN(hipSuccess); } @@ -1099,8 +1103,7 @@ hipError_t hipTexRefSetMipmappedArray(textureReference* texRef, resDesc.resType = hipResourceTypeMipmappedArray; resDesc.res.mipmap.mipmap = mipmappedArray; - // TODO need compiler support to extract the read mode from textureReference. - hipTextureDesc texDesc = hip::getTextureDesc(texRef, hipReadModeElementType); + hipTextureDesc texDesc = hip::getTextureDesc(texRef); hipResourceViewFormat format = hip::getResourceViewFormat(hip::getChannelFormatDesc(texRef->numChannels, texRef->format)); hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(mipmappedArray, format); From 4829a7c2156f5b87d9fb345fe63f4bed4f85f5db Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Wed, 18 Mar 2020 15:30:19 -0400 Subject: [PATCH 116/154] Add support for creating typed buffers What Cuda refers to "linear texture memory" is the OpenCL equivalent of CL_MEM_OBJECT_IMAGE1D_BUFFER. For these types of allocations we should create a typed buffer instead of an image. Currently there is no check in the texture fetch functions as to what kind of SRD is written into the texture object, so any kind of incorrect programming will cause the TA to hang. Fortunately for us, every one writes correct code :) Change-Id: I80dab85a992f2c0754ebf303d40ac6b5e045c7c1 --- include/hip/hcc_detail/ockl_image.h | 2 ++ include/hip/hcc_detail/texture_fetch_functions.h | 2 +- include/hip/hcc_detail/texture_indirect_functions.h | 2 +- vdi/hip_conversions.hpp | 2 +- vdi/hip_memory.cpp | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/hip/hcc_detail/ockl_image.h b/include/hip/hcc_detail/ockl_image.h index d9bc296791..b32b23fda0 100644 --- a/include/hip/hcc_detail/ockl_image.h +++ b/include/hip/hcc_detail/ockl_image.h @@ -30,6 +30,8 @@ extern "C" { __device__ float4::Native_vec_ __ockl_image_load_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c); +__device__ float4::Native_vec_ __ockl_image_load_1Db(unsigned int ADDRESS_SPACE_CONSTANT*i, int c); + __device__ float4::Native_vec_ __ockl_image_load_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c); __device__ float4::Native_vec_ __ockl_image_load_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c); diff --git a/include/hip/hcc_detail/texture_fetch_functions.h b/include/hip/hcc_detail/texture_fetch_functions.h index 23a850c52c..9e8e864520 100644 --- a/include/hip/hcc_detail/texture_fetch_functions.h +++ b/include/hip/hcc_detail/texture_fetch_functions.h @@ -139,7 +139,7 @@ template static __forceinline__ __device__ __hip_tex_ret_t tex1Dfetch(texture t, int x) { TEXTURE_PARAMETERS_INIT; - auto tmp = __ockl_image_sample_1D(i, s, x); + auto tmp = __ockl_image_load_1Db(i, x); return *reinterpret_cast<__hip_tex_ret_t*>(&tmp); } diff --git a/include/hip/hcc_detail/texture_indirect_functions.h b/include/hip/hcc_detail/texture_indirect_functions.h index 4facce436e..2fe33f3ede 100644 --- a/include/hip/hcc_detail/texture_indirect_functions.h +++ b/include/hip/hcc_detail/texture_indirect_functions.h @@ -65,7 +65,7 @@ template < static __device__ T tex1Dfetch(hipTextureObject_t textureObject, int x) { TEXTURE_OBJECT_PARAMETERS_INIT - auto tmp = __ockl_image_sample_1D(i, s, x); + auto tmp = __ockl_image_load_1Db(i, x); return *reinterpret_cast(&tmp); } diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index 0801c310eb..3b43cae2b1 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -153,7 +153,7 @@ inline cl_mem_object_type getCLMemObjectType(const hipResourceType hipResType) { switch (hipResType) { case hipResourceTypeLinear: - return CL_MEM_OBJECT_IMAGE1D; + return CL_MEM_OBJECT_IMAGE1D_BUFFER; case hipResourceTypePitch2D: return CL_MEM_OBJECT_IMAGE2D; default: diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 55bc11512a..6233c15a38 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -444,7 +444,7 @@ amd::Image* ihipImageCreate(const cl_channel_order channelOrder, amd::Image* image = nullptr; if (buffer != nullptr) { switch (imageType) { - case CL_MEM_OBJECT_IMAGE1D: + case CL_MEM_OBJECT_IMAGE1D_BUFFER: case CL_MEM_OBJECT_IMAGE2D: image = new (context) amd::Image(*buffer->asBuffer(), imageType, From 3baff106181f43fc9bb11f9558b3c4ccf57b5c19 Mon Sep 17 00:00:00 2001 From: Sarbojit Sarkar Date: Thu, 19 Mar 2020 10:42:21 +0530 Subject: [PATCH 117/154] Enable VDI Occupancy test Change-Id: Ia89155ca909b9b7346584d19962fa0a94033bb17 --- .../module/hipOccupancyMaxActiveBlocksPerMultiprocessor.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/src/runtimeApi/module/hipOccupancyMaxActiveBlocksPerMultiprocessor.cpp b/tests/src/runtimeApi/module/hipOccupancyMaxActiveBlocksPerMultiprocessor.cpp index 52c3fdbf96..ebf656b72f 100644 --- a/tests/src/runtimeApi/module/hipOccupancyMaxActiveBlocksPerMultiprocessor.cpp +++ b/tests/src/runtimeApi/module/hipOccupancyMaxActiveBlocksPerMultiprocessor.cpp @@ -22,7 +22,7 @@ THE SOFTWARE. // Test the Grid_Launch syntax. /* HIT_START - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc * TEST: %t * HIT_END */ @@ -46,10 +46,8 @@ int main(int argc, char* argv[]) { // test case for using kernel function pointer uint32_t gridSize = 0; uint32_t blockSize = 0; -#if defined(__HIP_PLATFORM_HCC__) && GENERIC_GRID_LAUNCH == 1 && defined(__HCC__) hipOccupancyMaxPotentialBlockSize(&gridSize, &blockSize, f1, 0, 0); assert(gridSize != 0 && blockSize != 0); -#endif uint32_t numBlock = 0; hipOccupancyMaxActiveBlocksPerMultiprocessor(&numBlock, f1, blockSize, 0); @@ -59,10 +57,8 @@ int main(int argc, char* argv[]) { // test case for using kernel function pointer with template gridSize = 0; blockSize = 0; -#if defined(__HIP_PLATFORM_HCC__) && GENERIC_GRID_LAUNCH == 1 && defined(__HCC__) hipOccupancyMaxPotentialBlockSize(&gridSize, &blockSize, f2, 0, 0); assert(gridSize != 0 && blockSize != 0); -#endif numBlock = 0; hipOccupancyMaxActiveBlocksPerMultiprocessor(&numBlock, f2, blockSize, 0); From 3dfbfc408ba16617f7d46240e680ef072ff4de0f Mon Sep 17 00:00:00 2001 From: Christophe Paquot Date: Thu, 19 Mar 2020 11:11:59 -0700 Subject: [PATCH 118/154] hipStreamAddCallback test seg faults Change-Id: I1f107fc8a5c586cd571f0280ed8716c5f89d25b7 SWDEV-227875: Need to add a dummy marker in case the stream is empty. --- vdi/hip_stream.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vdi/hip_stream.cpp b/vdi/hip_stream.cpp index 53311ae4ff..b66b525699 100644 --- a/vdi/hip_stream.cpp +++ b/vdi/hip_stream.cpp @@ -251,6 +251,11 @@ hipError_t hipStreamAddCallback(hipStream_t stream, hipStreamCallback_t callback amd::HostQueue* hostQueue = reinterpret_cast (stream)->asHostQueue(); amd::Command* command = hostQueue->getLastQueuedCommand(true); + if (command == nullptr) { + amd::Command::EventWaitList eventWaitList; + command = new amd::Marker(*hostQueue, false, eventWaitList); + command->enqueue(); + } amd::Event& event = command->event(); StreamCallback* cbo = new StreamCallback(stream, callback, userData, command); From cd92bd7feed141d24da4d7a50e2b24eb3993d3cd Mon Sep 17 00:00:00 2001 From: kjayapra-amd Date: Wed, 18 Mar 2020 13:43:58 -0400 Subject: [PATCH 119/154] SWDEV-216213 - Lookup module functions from PlatformState::functions_. Change-Id: I91dfe327f2ebdcf4c9b39ddd14d60aa0ce2fa9f4 --- vdi/hip_internal.hpp | 4 +++ vdi/hip_module.cpp | 53 ++++++++++++++++++++++++++---------- vdi/hip_platform.cpp | 65 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 105 insertions(+), 17 deletions(-) mode change 100644 => 100755 vdi/hip_internal.hpp mode change 100644 => 100755 vdi/hip_module.cpp mode change 100644 => 100755 vdi/hip_platform.cpp diff --git a/vdi/hip_internal.hpp b/vdi/hip_internal.hpp old mode 100644 new mode 100755 index e11e316503..b4c9f60e77 --- a/vdi/hip_internal.hpp +++ b/vdi/hip_internal.hpp @@ -194,6 +194,7 @@ public: std::string deviceName; std::vector< std::pair< hipModule_t, bool > >* modules; std::vector functions; + bool dyn_mod; }; struct DeviceVar { void* shadowVptr; @@ -216,6 +217,7 @@ public: return *platform_; } + bool unregisterFunc(hipModule_t hmod); std::vector< std::pair >* unregisterVar(hipModule_t hmod); @@ -223,6 +225,8 @@ public: void registerVar(const void* hostvar, const DeviceVar& var); void registerFunction(const void* hostFunction, const DeviceFunction& func); + bool findModFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name); + bool createFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name); hipFunction_t getFunc(const void* hostFunction, int deviceId); bool getFuncAttr(const void* hostFunction, hipFuncAttributes* func_attr); bool getGlobalVar(const void* hostVar, int deviceId, hipModule_t hmod, diff --git a/vdi/hip_module.cpp b/vdi/hip_module.cpp old mode 100644 new mode 100755 index b92155e518..0e22f9063b --- a/vdi/hip_module.cpp +++ b/vdi/hip_module.cpp @@ -92,6 +92,10 @@ hipError_t hipModuleUnload(hipModule_t hmod) amd::Program* program = as_amd(reinterpret_cast(hmod)); + if(!PlatformState::instance().unregisterFunc(hmod)) { + HIP_RETURN(hipErrorInvalidSymbol); + } + if(!ihipModuleUnregisterGlobal(hmod)) { HIP_RETURN(hipErrorInvalidSymbol); } @@ -122,7 +126,7 @@ extern hipError_t __hipExtractCodeObjectFromFatBinary(const void* data, const std::vector& devices, std::vector>& code_objs); -bool ihipModuleRegisterUndefined(amd::Program* program, hipModule_t* module) { +inline bool ihipModuleRegisterUndefined(amd::Program* program, hipModule_t* module) { std::vector undef_vars; device::Program* dev_program @@ -150,7 +154,34 @@ bool ihipModuleRegisterUndefined(amd::Program* program, hipModule_t* module) { return true; } -bool ihipModuleRegisterGlobal(amd::Program* program, hipModule_t* module) { +inline bool ihipModuleRegisterFunc(amd::Program* program, hipModule_t* module) { + + std::vector func_names; + device::Program* dev_program + = program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]); + + + // Get all the global func names from COMGR + if (!dev_program->getGlobalFuncFromCodeObj(&func_names)) { + return false; + } + + for (auto it = func_names.begin(); it != func_names.end(); ++it) { + auto modules = new std::vector >(g_devices.size()); + for (size_t dev = 0; dev < g_devices.size(); ++dev) { + modules->at(dev) = std::make_pair(*module, true); + } + + // Create a new pointer, since the hostFunction* wont be available + // if device code not in the same file as host code. + PlatformState::DeviceFunction dfunc{*it, modules, std::vector{ g_devices.size() }, true}; + PlatformState::instance().registerFunction(new std::string(it->c_str()), dfunc); + } + + return true; +} + +inline bool ihipModuleRegisterGlobal(amd::Program* program, hipModule_t* module) { size_t var_size = 0; hipDeviceptr_t device_ptr = nullptr; @@ -212,6 +243,10 @@ hipError_t ihipModuleLoadData(hipModule_t *module, const void *image) return hipErrorSharedObjectInitFailed; } + if (!ihipModuleRegisterFunc(program, module)) { + return hipErrorSharedObjectSymbolNotFound; + } + return hipSuccess; } @@ -219,21 +254,9 @@ hipError_t hipModuleGetFunction(hipFunction_t *hfunc, hipModule_t hmod, const ch { HIP_INIT_API(hipModuleGetFunction, hfunc, hmod, name); - amd::Program* program = as_amd(reinterpret_cast(hmod)); - - const amd::Symbol* symbol = program->findSymbol(name); - if (!symbol) { + if (!PlatformState::instance().findModFunc(hfunc, hmod, name)) { HIP_RETURN(hipErrorNotFound); } - - amd::Kernel* kernel = new amd::Kernel(*program, *symbol, name); - if (!kernel) { - HIP_RETURN(hipErrorOutOfMemory); - } - - hip::Function* f = new hip::Function(kernel); - *hfunc = f->asHipFunction(); - HIP_RETURN(hipSuccess); } diff --git a/vdi/hip_platform.cpp b/vdi/hip_platform.cpp old mode 100644 new mode 100755 index 4b0db00b1a..688181870c --- a/vdi/hip_platform.cpp +++ b/vdi/hip_platform.cpp @@ -188,6 +188,30 @@ void PlatformState::init() } } +bool PlatformState::unregisterFunc(hipModule_t hmod) { + amd::ScopedLock lock(lock_); + auto it = functions_.begin(); + while (it != functions_.end()) { + DeviceFunction& dfunc = it->second; + if ((*dfunc.modules)[0].first == hmod) { + if (dfunc.dyn_mod) { + std::string *s = reinterpret_cast(const_cast(it->first)); + delete s; + } + for (size_t dev = 0; dev < g_devices.size(); ++dev) { + if (dfunc.functions[dev] != 0) { + hip::Function* f = reinterpret_cast(dfunc.functions[dev]); + delete f; + } + } + functions_.erase(it++); + } else { + ++it; + } + } + return true; +} + std::vector< std::pair >* PlatformState::unregisterVar(hipModule_t hmod) { amd::ScopedLock lock(lock_); std::vector< std::pair >* rmodules = nullptr; @@ -292,6 +316,43 @@ bool CL_CALLBACK getSvarInfo(cl_program program, std::string var_name, void** va var_addr, var_size); } +bool PlatformState::findModFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name) { + amd::ScopedLock lock(lock_); + for (auto it = functions_.begin(); it != functions_.end(); ++it) { + PlatformState::DeviceFunction& devFunc = it->second; + if ((devFunc.deviceName == name) && (hmod == (*devFunc.modules)[ihipGetDevice()].first)) { + if (devFunc.functions[ihipGetDevice()] == 0) { + if(!createFunc(&devFunc.functions[ihipGetDevice()], hmod, name)) { + return false; + } + } + *hfunc = devFunc.functions[ihipGetDevice()]; + return true; + } + } + return false; +} + +bool PlatformState::createFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name) { + amd::Program* program = as_amd(reinterpret_cast(hmod)); + + const amd::Symbol* symbol = program->findSymbol(name); + if (!symbol) { + return false; + } + + amd::Kernel* kernel = new amd::Kernel(*program, *symbol, name); + if (!kernel) { + return false; + } + + hip::Function* f = new hip::Function(kernel); + *hfunc = f->asHipFunction(); + + return true; +} + + hipFunction_t PlatformState::getFunc(const void* hostFunction, int deviceId) { amd::ScopedLock lock(lock_); const auto it = functions_.find(hostFunction); @@ -308,7 +369,7 @@ hipFunction_t PlatformState::getFunc(const void* hostFunction, int deviceId) { (*devFunc.modules)[deviceId].second = true; } hipFunction_t function = nullptr; - if (hipSuccess == hipModuleGetFunction(&function, module, devFunc.deviceName.c_str()) && + if (createFunc(&function, module, devFunc.deviceName.c_str()) && function != nullptr) { devFunc.functions[deviceId] = function; } @@ -435,7 +496,7 @@ extern "C" void __hipRegisterFunction( dim3* gridDim, int* wSize) { - PlatformState::DeviceFunction func{ std::string{deviceName}, modules, std::vector{g_devices.size()}}; + PlatformState::DeviceFunction func{ std::string{deviceName}, modules, std::vector{g_devices.size()}, false}; PlatformState::instance().registerFunction(hostFunction, func); // for (size_t i = 0; i < g_devices.size(); ++i) { // PlatformState::instance().getFunc(hostFunction, i); From 8b39e0b74e1d8833cb2d0d0b6f04e369afed1d8a Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Fri, 20 Mar 2020 10:53:23 -0700 Subject: [PATCH 120/154] Sync streams when freeing or destroying mem Change-Id: I6932f225a8b932bb2adbd5e37880f7e604496809 --- vdi/hip_memory.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 6233c15a38..b38c2dbdbc 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -274,6 +274,13 @@ hipError_t hipHostFree(void* ptr) { HIP_INIT_API(hipHostFree, ptr); if (amd::SvmBuffer::malloced(ptr)) { + for (auto& dev : g_devices) { + amd::HostQueue* queue = hip::getNullStream(*dev->asContext()); + if (queue != nullptr) { + queue->finish(); + } + hip::syncStreams(dev->deviceId()); + } amd::SvmBuffer::free(*hip::getCurrentDevice()->asContext(), ptr); HIP_RETURN(hipSuccess); } @@ -289,7 +296,13 @@ hipError_t ihipArrayDestroy(hipArray* array) { if (is_valid(memObj) == false) { return hipErrorInvalidValue; } - + for (auto& dev : g_devices) { + amd::HostQueue* queue = hip::getNullStream(*dev->asContext()); + if (queue != nullptr) { + queue->finish(); + } + hip::syncStreams(dev->deviceId()); + } as_amd(memObj)->release(); delete array; From 653277bd3f6e3332f318423519b4dfce6b492ff5 Mon Sep 17 00:00:00 2001 From: Christophe Paquot Date: Fri, 20 Mar 2020 13:06:11 -0700 Subject: [PATCH 121/154] hipStreamAddCallback test seg faults Change-Id: If419d2fad490d0ed50eb1315af809fc1deda1ce3 SWDEV-227875: Add a lock in streams to lock when the callback is call so we make sure things aren't moving forward in the stream --- vdi/hip_internal.hpp | 2 +- vdi/hip_stream.cpp | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/vdi/hip_internal.hpp b/vdi/hip_internal.hpp index b4c9f60e77..129978dd68 100755 --- a/vdi/hip_internal.hpp +++ b/vdi/hip_internal.hpp @@ -129,7 +129,7 @@ namespace hip { struct Stream { amd::HostQueue* queue; - + amd::Monitor lock; Device* device; amd::CommandQueue::Priority priority; unsigned int flags; diff --git a/vdi/hip_stream.cpp b/vdi/hip_stream.cpp index b66b525699..eac42c0203 100644 --- a/vdi/hip_stream.cpp +++ b/vdi/hip_stream.cpp @@ -57,7 +57,7 @@ void syncStreams() { } Stream::Stream(hip::Device* dev, amd::CommandQueue::Priority p, unsigned int f) : - queue(nullptr), device(dev), priority(p), flags(f) {} + queue(nullptr), lock("Stream Callback lock"), device(dev), priority(p), flags(f) {} void Stream::create() { cl_command_queue_properties properties = CL_QUEUE_PROFILING_ENABLE; @@ -90,10 +90,12 @@ void Stream::finish() { }; void CL_CALLBACK ihipStreamCallback(cl_event event, cl_int command_exec_status, void* user_data) { - hipError_t status = hipSuccess; StreamCallback* cbo = reinterpret_cast(user_data); - cbo->callBack_(cbo->stream_, status, cbo->userData_); + { + amd::ScopedLock lock(reinterpret_cast(cbo->stream_)->lock); + cbo->callBack_(cbo->stream_, status, cbo->userData_); + } cbo->command_->release(); delete cbo; } From e7d91b8230a55c63ce18dc48f0d2d443a11d8508 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Mon, 23 Mar 2020 12:33:55 -0400 Subject: [PATCH 122/154] Disable failing tests for VDI The hipExtModuleLaunchKernel and hipModuleLoadDataMultiThreaded tests keeps randomly failing on Jenkins. Change-Id: I87e5d54fb7429c14ff1dcecb20e03a7816670fae --- tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp | 2 +- tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp b/tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp index f4c72ca1c5..a26c9be4a0 100755 --- a/tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp +++ b/tests/src/runtimeApi/module/hipExtModuleLaunchKernel.cpp @@ -19,7 +19,7 @@ THE SOFTWARE. /* HIT_START * BUILD_CMD: matmul.code %hc --genco %S/matmul.cpp -o matmul.code EXCLUDE_HIP_PLATFORM nvcc - * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc + * BUILD: %t %s ../../test_common.cpp EXCLUDE_HIP_PLATFORM nvcc vdi * TEST: %t * HIT_END */ diff --git a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp index e73bbedba5..4f180829fa 100644 --- a/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp +++ b/tests/src/runtimeApi/module/hipModuleLoadDataMultThreaded.cpp @@ -18,7 +18,7 @@ THE SOFTWARE. */ /* HIT_START - * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS -std=c++11 + * BUILD: %t %s ../../test_common.cpp NVCC_OPTIONS -std=c++11 EXCLUDE_HIP_PLATFORM vdi * TEST: %t * HIT_END */ From 76703acb1d116bbdbfff607ea28ad1d0b942d53b Mon Sep 17 00:00:00 2001 From: Payam Date: Wed, 25 Mar 2020 16:37:57 -0400 Subject: [PATCH 123/154] updated cmake to create libamdhip64 static file as well Change-Id: I2054b9501cefa232abbf398524ab62450ab6805d --- vdi/CMakeLists.txt | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/vdi/CMakeLists.txt b/vdi/CMakeLists.txt index 92332cb802..0bf319be47 100644 --- a/vdi/CMakeLists.txt +++ b/vdi/CMakeLists.txt @@ -8,6 +8,7 @@ if(CMAKE_CXX_FLAGS MATCHES "fsanitize=address") endif() set (CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(LIB_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib) set(CONFIG_PACKAGE_INSTALL_DIR ${LIB_INSTALL_DIR}/cmake/hip) @@ -114,7 +115,7 @@ include_directories("$" add_definitions(-DBSD_LIBELF) -add_library(amdhip64 SHARED +add_library(hip64 OBJECT hip_context.cpp hip_device.cpp hip_device_runtime.cpp @@ -135,9 +136,9 @@ add_library(amdhip64 SHARED cl_lqdflash_amd.cpp fixme.cpp ) - +set_target_properties(hip64 PROPERTIES POSITION_INDEPENDENT_CODE ON) set_target_properties( - amdhip64 PROPERTIES + hip64 PROPERTIES CXX_STANDARD 14 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF @@ -147,15 +148,25 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) include(${LIBVDI_STATIC_DIR}/amdvdi_staticTargets.cmake) +add_library(amdhip64 SHARED + $ + ) + +add_library(amdhip64_static STATIC + $ + ) + add_library(host INTERFACE) target_link_libraries(host INTERFACE amdhip64) +target_link_libraries(host INTERFACE amdhip64_static) add_library(device INTERFACE) target_link_libraries(device INTERFACE host) +target_link_libraries(amdhip64_static amdvdi_static pthread dl) target_link_libraries(amdhip64 amdvdi_static pthread dl) -#install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink libamdhip64.so lib/libhip_hcc.so)" ) +INSTALL(PROGRAMS $ DESTINATION lib COMPONENT MAIN) INSTALL(PROGRAMS $ DESTINATION lib COMPONENT MAIN) INSTALL(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink libamdhip64.so lib/libhip_hcc.so )" DESTINATION lib COMPONENT MAIN) @@ -164,6 +175,6 @@ INSTALL(FILES ${CMAKE_BINARY_DIR}/lib/libhip_hcc.so DESTINATION lib COMPONENT MA INSTALL(FILES ${CMAKE_BINARY_DIR}/lib/libhiprtc.so DESTINATION lib COMPONENT MAIN) -INSTALL(TARGETS amdhip64 host device EXPORT hip-targets DESTINATION ${LIB_INSTALL_DIR}) +INSTALL(TARGETS amdhip64_static amdhip64 host device EXPORT hip-targets DESTINATION ${LIB_INSTALL_DIR}) INSTALL(EXPORT hip-targets DESTINATION ${CONFIG_PACKAGE_INSTALL_DIR} NAMESPACE hip::) From 3774a1b1063ab317648d5cda8af50741e2a65023 Mon Sep 17 00:00:00 2001 From: jujiang Date: Thu, 19 Mar 2020 15:34:59 -0400 Subject: [PATCH 124/154] SWDEV-226708-hipMemcpyPeerAsync failed-->Update test app to create stream on proper device Change-Id: Ia9f51435ffb5a8fbbff39e363acecabb338bf34c --- tests/src/runtimeApi/memory/hipMemcpyPeerAsync.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/src/runtimeApi/memory/hipMemcpyPeerAsync.cpp b/tests/src/runtimeApi/memory/hipMemcpyPeerAsync.cpp index d08ff2a7e8..23ac329d64 100644 --- a/tests/src/runtimeApi/memory/hipMemcpyPeerAsync.cpp +++ b/tests/src/runtimeApi/memory/hipMemcpyPeerAsync.cpp @@ -60,16 +60,16 @@ int main() { HIPCHECK(hipDeviceSynchronize()); HipTest::checkVectorADD(A_h, B_h, C_h, N); - HIPCHECK(hipStreamCreate(&s)); HIPCHECK(hipSetDevice(1)); + HIPCHECK(hipStreamCreate(&s)); HIPCHECK(hipMemcpyPeerAsync(X_d, 1, A_d, 0, Nbytes, s)); HIPCHECK(hipMemcpyPeerAsync(Y_d, 1, B_d, 0, Nbytes, s)); hipLaunchKernelGGL(HipTest::vectorADD, dim3(blocks), dim3(threadsPerBlock), 0, 0, static_cast(X_d), static_cast(Y_d), Z_d, N); HIPCHECK(hipMemcpy(C_h, Z_d, Nbytes, hipMemcpyDeviceToHost)); - HIPCHECK(hipDeviceSynchronize()); HIPCHECK(hipStreamSynchronize(s)); + HIPCHECK(hipDeviceSynchronize()); HipTest::checkVectorADD(A_h, B_h, C_h, N); HIPCHECK(hipStreamDestroy(s)); From ea701777d8fa8716b77547c150508d79574adb73 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Thu, 19 Mar 2020 16:05:06 -0400 Subject: [PATCH 125/154] Allow creating texture from unaligned user ptr All we have to do is align the ptr to HW requirments an if it's not zero, then return the offset to the user. We currently don't have anywhere to store this offset, so hipGetTextureAlignmentOffset() will still always return 0. Change-Id: If31998127d99a2a3222a026d88249519d6102505 --- vdi/hip_texture.cpp | 64 ++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 2d4085d928..b2e58a7f69 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -382,6 +382,28 @@ hipError_t hipGetTextureObjectTextureDesc(hipTextureDesc* pTexDesc, HIP_RETURN(hipSuccess); } +inline bool ihipGetTextureAlignmentOffset(size_t* offset, + const void* devPtr) { + amd::Device* device = hip::getCurrentDevice()->devices()[0]; + const device::Info& info = device->info(); + + const char* alignedDevPtr = amd::alignUp(static_cast(devPtr), info.imageBaseAddressAlignment_); + const size_t alignedOffset = alignedDevPtr - static_cast(devPtr); + + // If the device memory pointer was returned from hipMalloc(), + // the offset is guaranteed to be 0 and NULL may be passed as the offset parameter. + if ((alignedOffset != 0) && + (offset == nullptr)) { + return false; + } + + if (offset != nullptr) { + *offset = alignedOffset; + } + + return true; +} + hipError_t ihipBindTexture(size_t* offset, const textureReference* texref, const void* devPtr, @@ -398,19 +420,19 @@ hipError_t ihipBindTexture(size_t* offset, // No need to check for errors. ihipDestroyTextureObject(texref->textureObject); - // If the device memory pointer was returned from hipMalloc(), - // the offset is guaranteed to be 0 and NULL may be passed as the offset parameter. - // TODO enforce alignment on devPtr. - if (offset != nullptr) { - *offset = 0; - } - hipResourceDesc resDesc = {}; resDesc.resType = hipResourceTypeLinear; resDesc.res.linear.devPtr = const_cast(devPtr); resDesc.res.linear.desc = *desc; resDesc.res.linear.sizeInBytes = size; + if (ihipGetTextureAlignmentOffset(offset, devPtr)) { + // Align the user ptr to HW requirments. + resDesc.res.linear.devPtr = static_cast(const_cast(devPtr)) - *offset; + } else { + return hipErrorInvalidValue; + } + hipTextureDesc texDesc = hip::getTextureDesc(texref); return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); @@ -434,13 +456,6 @@ hipError_t ihipBindTexture2D(size_t* offset, // No need to check for errors. ihipDestroyTextureObject(texref->textureObject); - // If the device memory pointer was returned from hipMalloc(), - // the offset is guaranteed to be 0 and NULL may be passed as the offset parameter. - // TODO enforce alignment on devPtr. - if (offset != nullptr) { - *offset = 0; - } - hipResourceDesc resDesc = {}; resDesc.resType = hipResourceTypePitch2D; resDesc.res.pitch2D.devPtr = const_cast(devPtr); @@ -449,6 +464,13 @@ hipError_t ihipBindTexture2D(size_t* offset, resDesc.res.pitch2D.height = height; resDesc.res.pitch2D.pitchInBytes = pitch; + if (ihipGetTextureAlignmentOffset(offset, devPtr)) { + // Align the user ptr to HW requirments. + resDesc.res.pitch2D.devPtr = static_cast(const_cast(devPtr)) - *offset; + } else { + return hipErrorInvalidValue; + } + hipTextureDesc texDesc = hip::getTextureDesc(texref); return ihipCreateTextureObject(const_cast(&texref->textureObject), &resDesc, &texDesc, nullptr); @@ -798,19 +820,19 @@ hipError_t hipTexRefSetAddress(size_t* ByteOffset, // No need to check for errors. ihipDestroyTextureObject(texRef->textureObject); - // If the device memory pointer was returned from hipMemAlloc(), - // the offset is guaranteed to be 0 and NULL may be passed as the ByteOffset parameter. - // TODO enforce alignment on devPtr. - if (ByteOffset != nullptr) { - *ByteOffset = 0; - } - hipResourceDesc resDesc = {}; resDesc.resType = hipResourceTypeLinear; resDesc.res.linear.devPtr = dptr; resDesc.res.linear.desc = hip::getChannelFormatDesc(texRef->numChannels, texRef->format); resDesc.res.linear.sizeInBytes = bytes; + if (ihipGetTextureAlignmentOffset(ByteOffset, dptr)) { + // Align the user ptr to HW requirments. + resDesc.res.linear.devPtr = static_cast(dptr) - *ByteOffset; + } else { + return HIP_RETURN(hipErrorInvalidValue); + } + hipTextureDesc texDesc = hip::getTextureDesc(texRef); HIP_RETURN(ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, nullptr)); From b09fe1280eee0658f18e54322fa8ad02df7d929b Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Thu, 19 Mar 2020 19:17:30 -0400 Subject: [PATCH 126/154] Correct typos in texture function declarations Change-Id: I492995e984eda2e8a5e806c5d4c9c78da09ac483 --- include/hip/hcc_detail/texture_fetch_functions.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/hip/hcc_detail/texture_fetch_functions.h b/include/hip/hcc_detail/texture_fetch_functions.h index 9e8e864520..03c1780030 100644 --- a/include/hip/hcc_detail/texture_fetch_functions.h +++ b/include/hip/hcc_detail/texture_fetch_functions.h @@ -200,7 +200,7 @@ static __forceinline__ __device__ __hip_tex_ret_t tex1DLod(texture< } template -static __forceinline__ __device__ __hip_tex_ret_t tex2DLod(texture t, float x, float y, float level) +static __forceinline__ __device__ __hip_tex_ret_t tex2DLod(texture t, float x, float y, float level) { TEXTURE_PARAMETERS_INIT; auto tmp = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level); @@ -240,7 +240,7 @@ static __forceinline__ __device__ __hip_tex_ret_t texCubemapLod(tex } template -static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayered(texture t, float x, float y, float z, int layer) +static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayered(texture t, float x, float y, float z, int layer) { TEXTURE_PARAMETERS_INIT; auto tmp = __ockl_image_sample_CMa(i, s, float4(x, y, z, layer).data); @@ -248,7 +248,7 @@ static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayered } template -static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayeredLod(texture t, float x, float y, float z, int layer, float level) +static __forceinline__ __device__ __hip_tex_ret_t texCubemapLayeredLod(texture t, float x, float y, float z, int layer, float level) { TEXTURE_PARAMETERS_INIT; auto tmp = __ockl_image_sample_lod_CMa(i, s, float4(x, y, z, layer).data, level); @@ -300,7 +300,7 @@ static __forceinline__ __device__ __hip_tex_ret_t tex1DLayeredGrad( } template -static __forceinline__ __device__ __hip_tex_ret_t tex2DLayeredGrad(texture t, float x, float y, int layer, float2 dPdx, float2 dPdy) +static __forceinline__ __device__ __hip_tex_ret_t tex2DLayeredGrad(texture t, float x, float y, int layer, float2 dPdx, float2 dPdy) { TEXTURE_PARAMETERS_INIT; auto tmp = __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data); From 39558dc1b9eea76e8bab3c240068ebee70cd515f Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 20 Mar 2020 15:10:43 -0400 Subject: [PATCH 127/154] Modify formatting for textureReferences We don't program the numChannels and format members (these are HCC specific), so printing these will only display garbage. Change-Id: I83dc8be9a3cae2659c64f4594d07c05330d2dd14 --- vdi/hip_formatting.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vdi/hip_formatting.hpp b/vdi/hip_formatting.hpp index 1e1c02b25f..d75c2071fc 100644 --- a/vdi/hip_formatting.hpp +++ b/vdi/hip_formatting.hpp @@ -712,10 +712,6 @@ inline std::ostream& operator<<(std::ostream& os, const textureReference& s) { << s.maxMipmapLevelClamp << ',' << s.textureObject - << ',' - << s.numChannels - << ',' - << s.format << '}'; return os; } From ced0582a526646cb815c4889de79bd65aaaa5138 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 20 Mar 2020 15:11:39 -0400 Subject: [PATCH 128/154] Set textureObject to nullptr This avoids dangling pointers for newly initiazlied textures Change-Id: Ia444b91fe17fd756ed583ec595ae1febbdfbd034 --- include/hip/hcc_detail/hip_texture_types.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/hip/hcc_detail/hip_texture_types.h b/include/hip/hcc_detail/hip_texture_types.h index e82ef30f00..e92babfd5a 100644 --- a/include/hip/hcc_detail/hip_texture_types.h +++ b/include/hip/hcc_detail/hip_texture_types.h @@ -64,6 +64,7 @@ struct __HIP_TEXTURE_ATTRIB texture : public textureReference { addressMode[2] = aMode; channelDesc = hipCreateChannelDesc(); sRGB = 0; + textureObject = nullptr; } texture(int norm, enum hipTextureFilterMode fMode, enum hipTextureAddressMode aMode, @@ -76,6 +77,7 @@ struct __HIP_TEXTURE_ATTRIB texture : public textureReference { addressMode[2] = aMode; channelDesc = desc; sRGB = 0; + textureObject = nullptr; } }; From 2028b6eb29c9bc50e7033e35f449b0a1ac8d93e4 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Mon, 23 Mar 2020 15:41:28 -0400 Subject: [PATCH 129/154] Headers need to export C symbols for texture API This also adds declarations of all the missing texture APIs. hipTexRefSet*() functions need to take a textureReference as a ptr for type erasure to work. Runtime has been modified to accomodate this. This change only applies to VDI. Change-Id: Icf43cc5bd44dfc2c39084b7fe56d5a793bf7319f --- include/hip/hcc_detail/channel_descriptor.h | 6 + include/hip/hcc_detail/hip_runtime_api.h | 199 +++++++++++++++++--- vdi/hip_hcc.def.in | 44 +++-- vdi/hip_hcc.map.in | 64 ++++--- vdi/hip_texture.cpp | 116 +++++++----- 5 files changed, 313 insertions(+), 116 deletions(-) diff --git a/include/hip/hcc_detail/channel_descriptor.h b/include/hip/hcc_detail/channel_descriptor.h index 38acff9951..a69558c8e4 100644 --- a/include/hip/hcc_detail/channel_descriptor.h +++ b/include/hip/hcc_detail/channel_descriptor.h @@ -29,8 +29,14 @@ THE SOFTWARE. #ifdef __cplusplus +#if __HIP_VDI__ +extern "C" { +#endif HIP_PUBLIC_API hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f); +#if __HIP_VDI__ +} +#endif static inline hipChannelFormatDesc hipCreateChannelDescHalf() { int e = (int)sizeof(unsigned short) * 8; diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 2a781154e1..d6b66090cd 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -3314,6 +3314,172 @@ hipError_t hipLaunchKernel(const void* function_address, size_t sharedMemBytes __dparm(0), hipStream_t stream __dparm(0)); +#if __HIP_VDI__ +hipError_t hipBindTexture( + size_t* offset, + const textureReference* tex, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t size = UINT_MAX); + +hipError_t hipBindTexture2D( + size_t* offset, + const textureReference* tex, + const void* devPtr, + const hipChannelFormatDesc* desc, + size_t width, + size_t height, + size_t pitch); + +hipError_t hipBindTextureToArray( + const textureReference* tex, + hipArray_const_t array, + const hipChannelFormatDesc* desc); + +hipError_t hipBindTextureToMipmappedArray( + const textureReference* tex, + hipMipmappedArray_const_t mipmappedArray, + const hipChannelFormatDesc* desc); + +hipError_t hipGetTextureAlignmentOffset( + size_t* offset, + const textureReference* texref); + +hipError_t hipGetTextureReference( + const textureReference** texref, + const void* symbol); + +hipError_t hipUnbindTexture(const textureReference* tex); + +hipError_t hipCreateTextureObject( + hipTextureObject_t* pTexObject, + const hipResourceDesc* pResDesc, + const hipTextureDesc* pTexDesc, + const hipResourceViewDesc* pResViewDesc); + +hipError_t hipDestroyTextureObject(hipTextureObject_t textureObject); + +hipError_t hipGetChannelDesc( + hipChannelFormatDesc* desc, + hipArray_const_t array); + +hipError_t hipGetTextureObjectResourceDesc( + hipResourceDesc* pResDesc, + hipTextureObject_t textureObject); + +hipError_t hipGetTextureObjectResourceViewDesc( + hipResourceViewDesc* pResViewDesc, + hipTextureObject_t textureObject); + +hipError_t hipGetTextureObjectTextureDesc( + hipTextureDesc* pTexDesc, + hipTextureObject_t textureObject); + +hipError_t hipTexRefGetAddress( + hipDeviceptr_t* dev_ptr, + const textureReference* texRef); + +hipError_t hipTexRefGetAddressMode( + hipTextureAddressMode* pam, + const textureReference* texRef, + int dim); + +hipError_t hipTexRefGetFilterMode( + hipTextureFilterMode* pfm, + const textureReference* texRef); + +hipError_t hipTexRefGetFlags( + unsigned int* pFlags, + const textureReference* texRef); + +hipError_t hipTexRefGetFormat( + hipArray_Format* pFormat, + int* pNumChannels, + const textureReference* texRef); + +hipError_t hipTexRefGetMaxAnisotropy( + int* pmaxAnsio, + const textureReference* texRef); + +hipError_t hipTexRefGetMipmapFilterMode( + hipTextureFilterMode* pfm, + const textureReference* texRef); + +hipError_t hipTexRefGetMipmapLevelBias( + float* pbias, + const textureReference* texRef); + +hipError_t hipTexRefGetMipmapLevelClamp( + float* pminMipmapLevelClamp, + float* pmaxMipmapLevelClamp, + const textureReference* texRef); + +hipError_t hipTexRefGetMipMappedArray( + hipMipmappedArray_t* pArray, + const textureReference* texRef); + +hipError_t hipTexRefSetAddress( + size_t* ByteOffset, + textureReference* texRef, + hipDeviceptr_t dptr, + size_t bytes); + +hipError_t hipTexRefSetAddress2D( + textureReference* texRef, + const HIP_ARRAY_DESCRIPTOR* desc, + hipDeviceptr_t dptr, + size_t Pitch); + +hipError_t hipTexRefSetAddressMode( + textureReference* texRef, + int dim, + hipTextureAddressMode am); + +hipError_t hipTexRefSetArray( + textureReference* tex, + hipArray_const_t array, + unsigned int flags); + +hipError_t hipTexRefSetBorderColor( + textureReference* texRef, + float* pBorderColor); + +hipError_t hipTexRefSetFilterMode( + textureReference* texRef, + hipTextureFilterMode fm); + +hipError_t hipTexRefSetFlags( + textureReference* texRef, + unsigned int Flags); + +hipError_t hipTexRefSetFormat( + textureReference* texRef, + hipArray_Format fmt, + int NumPackedComponents); + +hipError_t hipTexRefSetMaxAnisotropy( + textureReference* texRef, + unsigned int maxAniso); + +hipError_t hipTexRefSetMipmapFilterMode( + textureReference* texRef, + hipTextureFilterMode fm); + +hipError_t hipTexRefSetMipmapLevelBias( + textureReference* texRef, + float bias); + +hipError_t hipTexRefSetMipmapLevelClamp( + textureReference* texRef, + float minMipMapLevelClamp, + float maxMipMapLevelClamp); + +hipError_t hipTexRefSetMipmappedArray( + textureReference* texRef, + hipMipmappedArray* mipmappedArray, + unsigned int Flags); +#endif + /** * @} */ @@ -3362,14 +3528,7 @@ const char* hipKernelNameRef(const hipFunction_t f); class TlsData; -#if __HIP_VDI__ -hipError_t hipBindTexture( - size_t* offset, - const textureReference* tex, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t size = UINT_MAX); -#else +#if !__HIP_VDI__ hipError_t hipBindTexture(size_t* offset, textureReference* tex, const void* devPtr, const hipChannelFormatDesc* desc, size_t size = UINT_MAX); #endif @@ -3426,16 +3585,7 @@ hipError_t hipBindTexture(size_t* offset, struct texture& tex, #endif // C API -#if __HIP_VDI__ -hipError_t hipBindTexture2D( - size_t* offset, - const textureReference* tex, - const void* devPtr, - const hipChannelFormatDesc* desc, - size_t width, - size_t height, - size_t pitch); -#else +#if !__HIP_VDI__ hipError_t hipBindTexture2D(size_t* offset, textureReference* tex, const void* devPtr, const hipChannelFormatDesc* desc, size_t width, size_t height, size_t pitch); @@ -3466,12 +3616,7 @@ hipError_t hipBindTexture2D(size_t* offset, struct texture& te #endif // C API -#if __HIP_VDI__ -hipError_t hipBindTextureToArray( - const textureReference* tex, - hipArray_const_t array, - const hipChannelFormatDesc* desc); -#else +#if !__HIP_VDI__ hipError_t hipBindTextureToArray(textureReference* tex, hipArray_const_t array, const hipChannelFormatDesc* desc); #endif @@ -3508,9 +3653,11 @@ inline static hipError_t hipBindTextureToArray(struct texture #endif // C API +#if !__HIP_VDI__ hipError_t hipBindTextureToMipmappedArray(const textureReference* tex, hipMipmappedArray_const_t mipmappedArray, const hipChannelFormatDesc* desc); +#endif #if !__HIP_VDI__ template @@ -3566,7 +3713,9 @@ inline hipError_t hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchPara * * @return #hipSuccess **/ +#if !__HIP_VDI__ hipError_t hipUnbindTexture(const textureReference* tex); +#endif #if !__HIP_VDI__ extern hipError_t ihipUnbindTextureImpl(const hipTextureObject_t& textureObject); @@ -3579,6 +3728,7 @@ hipError_t hipUnbindTexture(struct texture& tex) { } #endif +#if !__HIP_VDI__ hipError_t hipGetChannelDesc(hipChannelFormatDesc* desc, hipArray_const_t array); hipError_t hipGetTextureAlignmentOffset(size_t* offset, const textureReference* texref); hipError_t hipGetTextureReference(const textureReference** texref, const void* symbol); @@ -3616,6 +3766,7 @@ hipError_t hipTexRefGetAddress(hipDeviceptr_t* dev_ptr, textureReference tex); hipError_t hipTexRefSetAddress2D(textureReference* tex, const HIP_ARRAY_DESCRIPTOR* desc, hipDeviceptr_t devPtr, size_t pitch); +#endif hipError_t hipCreateSurfaceObject(hipSurfaceObject_t* pSurfObject, const hipResourceDesc* pResDesc); diff --git a/vdi/hip_hcc.def.in b/vdi/hip_hcc.def.in index 57d3a80719..c6e4a00200 100644 --- a/vdi/hip_hcc.def.in +++ b/vdi/hip_hcc.def.in @@ -173,31 +173,45 @@ hipRegisterActivityCallback hipRemoveActivityCallback hipApiName hipKernelNameRef -hipCreateTextureObject -hipDestroyTextureObject -hipGetTextureObjectResourceDesc -hipGetTextureObjectResourceViewDesc -hipGetTextureObjectTextureDesc hipBindTexture hipBindTexture2D hipBindTextureToArray hipBindTextureToMipmappedArray -hipUnbindTexture -hipGetChannelDesc hipGetTextureAlignmentOffset hipGetTextureReference -hipTexRefSetFormat -hipTexRefSetFlags -hipTexRefSetFilterMode -hipTexRefSetAddressMode -hipTexRefSetArray -hipTexRefSetAddress -hipTexRefSetAddress2D +hipUnbindTexture +hipCreateChannelDesc +hipCreateTextureObject +hipDestroyTextureObject +hipGetChannelDesc +hipGetTextureObjectResourceDesc +hipGetTextureObjectResourceViewDesc +hipGetTextureObjectTextureDesc hipTexRefGetAddress hipTexRefGetAddressMode hipTexRefGetArray +hipTexRefGetBorderColor +hipTexRefGetFilterMode +hipTexRefGetFlags +hipTexRefGetFormat +hipTexRefGetMaxAnisotropy +hipTexRefGetMipmapFilterMode +hipTexRefGetMipmapLevelBias +hipTexRefGetMipmapLevelClamp +hipTexRefGetMipmappedArray +hipTexRefSetAddress +hipTexRefSetAddress2D +hipTexRefSetAddressMode hipTexRefSetArray -hipCreateChannelDesc +hipTexRefSetBorderColor +hipTexRefSetFilterMode +hipTexRefSetFlags +hipTexRefSetFormat +hipTexRefSetMaxAnisotropy +hipTexRefSetMipmapFilterMode +hipTexRefSetMipmapLevelBias +hipTexRefSetMipmapLevelClamp +hipTexRefSetMipmappedArray hipProfilerStart hipProfilerStop hipHccGetAccelerator diff --git a/vdi/hip_hcc.map.in b/vdi/hip_hcc.map.in index 70109c4943..c26e79fc70 100644 --- a/vdi/hip_hcc.map.in +++ b/vdi/hip_hcc.map.in @@ -186,34 +186,48 @@ global: hiprtcGetErrorString; hiprtcAddNameExpression; hiprtcVersion; + hipBindTexture; + hipBindTexture2D; + hipBindTextureToArray; + hipBindTextureToMipmappedArray; + hipGetTextureAlignmentOffset; + hipGetTextureReference; + hipUnbindTexture; + hipCreateChannelDesc; + hipCreateTextureObject; + hipDestroyTextureObject; + hipGetChannelDesc; + hipGetTextureObjectResourceDesc; + hipGetTextureObjectResourceViewDesc; + hipGetTextureObjectTextureDesc; + hipTexRefGetAddress; + hipTexRefGetAddressMode; + hipTexRefGetArray; + hipTexRefGetBorderColor; + hipTexRefGetFilterMode; + hipTexRefGetFlags; + hipTexRefGetFormat; + hipTexRefGetMaxAnisotropy; + hipTexRefGetMipmapFilterMode; + hipTexRefGetMipmapLevelBias; + hipTexRefGetMipmapLevelClamp; + hipTexRefGetMipmappedArray; + hipTexRefSetAddress; + hipTexRefSetAddress2D; + hipTexRefSetAddressMode; + hipTexRefSetArray; + hipTexRefSetBorderColor; + hipTexRefSetFilterMode; + hipTexRefSetFlags; + hipTexRefSetFormat; + hipTexRefSetMaxAnisotropy; + hipTexRefSetMipmapFilterMode; + hipTexRefSetMipmapLevelBias; + hipTexRefSetMipmapLevelClamp; + hipTexRefSetMipmappedArray; extern "C++" { hip_impl::hipLaunchKernelGGLImpl*; hip_impl::demangle*; - hipCreateTextureObject*; - hipDestroyTextureObject*; - hipGetTextureObjectResourceDesc*; - hipGetTextureObjectResourceViewDesc*; - hipGetTextureObjectTextureDesc*; - hipBindTexture*; - hipBindTexture2D*; - hipBindTextureToArray*; - hipBindTextureToMipmappedArray*; - hipUnbindTexture*; - hipGetChannelDesc*; - hipGetTextureAlignmentOffset*; - hipGetTextureReference*; - hipTexRefSetFormat*; - hipTexRefSetFlags*; - hipTexRefSetFilterMode*; - hipTexRefSetAddressMode*; - hipTexRefSetArray*; - hipTexRefSetAddress*; - hipTexRefSetAddress2D*; - hipTexRefGetAddress*; - hipTexRefGetAddressMode*; - hipTexRefGetArray*; - hipTexRefSetArray*; - hipCreateChannelDesc*; hipHccGetAccelerator*; hipHccGetAcceleratorView*; hipCreateSurfaceObject*; diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 6b2f151922..19761d1e78 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -671,12 +671,13 @@ hipError_t hipTexRefSetFilterMode(textureReference* texRef, } hipError_t hipTexRefGetAddressMode(hipTextureAddressMode* pam, - textureReference texRef, + const textureReference* texRef, int dim) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetAddressMode, pam, &texRef, dim); + HIP_INIT_API(hipTexRefGetAddressMode, pam, texRef, dim); - if (pam == nullptr) { + if ((pam == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } @@ -685,7 +686,7 @@ hipError_t hipTexRefGetAddressMode(hipTextureAddressMode* pam, HIP_RETURN(hipErrorInvalidValue); } - *pam = texRef.addressMode[dim]; + *pam = texRef->addressMode[dim]; HIP_RETURN(hipSuccess); } @@ -709,17 +710,18 @@ hipError_t hipTexRefSetAddressMode(textureReference* texRef, } hipError_t hipTexRefGetArray(hipArray_t* pArray, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetArray, pArray, &texRef); + HIP_INIT_API(hipTexRefGetArray, pArray, texRef); - if (pArray == nullptr) { + if ((pArray == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } hipResourceDesc resDesc = {}; // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. - hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef.textureObject); + hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef->textureObject); if (error != hipSuccess) { return HIP_RETURN(error); } @@ -769,17 +771,18 @@ hipError_t hipTexRefSetArray(textureReference* texRef, } hipError_t hipTexRefGetAddress(hipDeviceptr_t* dptr, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetAddress, dptr, &texRef); + HIP_INIT_API(hipTexRefGetAddress, dptr, texRef); - if (dptr == nullptr) { + if ((dptr == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } hipResourceDesc resDesc = {}; // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. - hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef.textureObject); + hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef->textureObject); if (error != hipSuccess) { return HIP_RETURN(error); } @@ -869,11 +872,12 @@ hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannel } hipError_t hipTexRefGetBorderColor(float* pBorderColor, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetBorderColor, pBorderColor, &texRef); + HIP_INIT_API(hipTexRefGetBorderColor, pBorderColor, texRef); - if (pBorderColor == nullptr) { + if ((pBorderColor == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } @@ -885,39 +889,41 @@ hipError_t hipTexRefGetBorderColor(float* pBorderColor, } hipError_t hipTexRefGetFilterMode(hipTextureFilterMode* pfm, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetFilterMode, pfm, &texRef); + HIP_INIT_API(hipTexRefGetFilterMode, pfm, texRef); - if (pfm == nullptr) { + if ((pfm == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } - *pfm = texRef.filterMode; + *pfm = texRef->filterMode; HIP_RETURN(hipSuccess); } hipError_t hipTexRefGetFlags(unsigned int* pFlags, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetFlags, pFlags, &texRef); + HIP_INIT_API(hipTexRefGetFlags, pFlags, texRef); - if (pFlags == nullptr) { + if ((pFlags == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } *pFlags = 0; - if (texRef.readMode == hipReadModeElementType) { + if (texRef->readMode == hipReadModeElementType) { *pFlags |= HIP_TRSF_READ_AS_INTEGER; } - if (texRef.normalized == 1) { + if (texRef->normalized == 1) { *pFlags |= HIP_TRSF_NORMALIZED_COORDINATES; } - if (texRef.sRGB == 1) { + if (texRef->sRGB == 1) { *pFlags |= HIP_TRSF_SRGB; } @@ -926,92 +932,98 @@ hipError_t hipTexRefGetFlags(unsigned int* pFlags, hipError_t hipTexRefGetFormat(hipArray_Format* pFormat, int* pNumChannels, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetFormat, pFormat, pNumChannels, &texRef); + HIP_INIT_API(hipTexRefGetFormat, pFormat, pNumChannels, texRef); if ((pFormat == nullptr) || - (pNumChannels == nullptr)) { + (pNumChannels == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } - *pFormat = texRef.format; - *pNumChannels = texRef.numChannels; + *pFormat = texRef->format; + *pNumChannels = texRef->numChannels; HIP_RETURN(hipSuccess); } hipError_t hipTexRefGetMaxAnisotropy(int* pmaxAnsio, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMaxAnisotropy, pmaxAnsio, &texRef); + HIP_INIT_API(hipTexRefGetMaxAnisotropy, pmaxAnsio, texRef); - if (pmaxAnsio == nullptr) { + if ((pmaxAnsio == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } - *pmaxAnsio = texRef.maxAnisotropy; + *pmaxAnsio = texRef->maxAnisotropy; HIP_RETURN(hipErrorInvalidValue); } hipError_t hipTexRefGetMipmapFilterMode(hipTextureFilterMode* pfm, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMipmapFilterMode, pfm, &texRef); + HIP_INIT_API(hipTexRefGetMipmapFilterMode, pfm, texRef); - if (pfm == nullptr) { + if ((pfm == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } - *pfm = texRef.mipmapFilterMode; + *pfm = texRef->mipmapFilterMode; HIP_RETURN(hipErrorInvalidValue); } hipError_t hipTexRefGetMipmapLevelBias(float* pbias, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMipmapLevelBias, pbias, &texRef); + HIP_INIT_API(hipTexRefGetMipmapLevelBias, pbias, texRef); - if (pbias == nullptr) { + if ((pbias == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } - *pbias = texRef.mipmapLevelBias; + *pbias = texRef->mipmapLevelBias; HIP_RETURN(hipErrorInvalidValue); } hipError_t hipTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, float* pmaxMipmapLevelClamp, - textureReference texRef) { + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMipmapLevelClamp, pminMipmapLevelClamp, pmaxMipmapLevelClamp, &texRef); + HIP_INIT_API(hipTexRefGetMipmapLevelClamp, pminMipmapLevelClamp, pmaxMipmapLevelClamp, texRef); if ((pminMipmapLevelClamp == nullptr) || - (pmaxMipmapLevelClamp == nullptr)){ + (pmaxMipmapLevelClamp == nullptr) || + (texRef == nullptr)){ HIP_RETURN(hipErrorInvalidValue); } - *pminMipmapLevelClamp = texRef.minMipmapLevelClamp; - *pmaxMipmapLevelClamp = texRef.maxMipmapLevelClamp; + *pminMipmapLevelClamp = texRef->minMipmapLevelClamp; + *pmaxMipmapLevelClamp = texRef->maxMipmapLevelClamp; HIP_RETURN(hipErrorInvalidValue); } -hipError_t hipTexRefGetMipMappedArray(hipMipmappedArray_t* pArray, - textureReference texRef) { +hipError_t hipTexRefGetMipmappedArray(hipMipmappedArray_t* pArray, + const textureReference* texRef) { // TODO overload operator<<(ostream&, textureReference&). - HIP_INIT_API(hipTexRefGetMipMappedArray, pArray, &texRef); + HIP_INIT_API(hipTexRefGetMipmappedArray, pArray, &texRef); - if (pArray == nullptr) { + if ((pArray == nullptr) || + (texRef == nullptr)) { HIP_RETURN(hipErrorInvalidValue); } hipResourceDesc resDesc = {}; // TODO use ihipGetTextureObjectResourceDesc() to not pollute the API trace. - hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef.textureObject); + hipError_t error = hipGetTextureObjectResourceDesc(&resDesc, texRef->textureObject); if (error != hipSuccess) { return HIP_RETURN(error); } From fbe30090a2407d28becbbbc64a22646dca6d1d62 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 20 Mar 2020 16:29:46 -0400 Subject: [PATCH 130/154] Correctly format hipResourceDesc objects The struct consists of a union - only the active object should be read. Change-Id: I1c40965b61518acd91a2dcbae92a015ac9be346a --- vdi/hip_formatting.hpp | 53 ++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/vdi/hip_formatting.hpp b/vdi/hip_formatting.hpp index d75c2071fc..ec92b137c1 100644 --- a/vdi/hip_formatting.hpp +++ b/vdi/hip_formatting.hpp @@ -614,26 +614,39 @@ inline std::ostream& operator<<(std::ostream& os, const hipResourceDesc& s) { os << '{' << s.resType << ',' - << s.res.array.array - << ',' - << s.res.mipmap.mipmap - << ',' - << s.res.linear.devPtr - << ',' - << s.res.linear.desc - << ',' - << s.res.linear.sizeInBytes - << ',' - << s.res.pitch2D.devPtr - << ',' - << s.res.pitch2D.desc - << ',' - << s.res.pitch2D.width - << ',' - << s.res.pitch2D.height - << ',' - << s.res.pitch2D.pitchInBytes - << '}'; + << '{'; + + switch (s.resType) { + case hipResourceTypeLinear: + os << s.res.linear.devPtr + << ',' + << s.res.linear.desc + << ',' + << s.res.linear.sizeInBytes; + break; + case hipResourceTypePitch2D: + os << s.res.pitch2D.devPtr + << ',' + << s.res.pitch2D.desc + << ',' + << s.res.pitch2D.width + << ',' + << s.res.pitch2D.height + << ',' + << s.res.pitch2D.pitchInBytes; + break; + case hipResourceTypeArray: + os << s.res.array.array; + break; + case hipResourceTypeMipmappedArray: + os < Date: Mon, 23 Mar 2020 15:53:29 -0400 Subject: [PATCH 131/154] Replace hip::TextureObject with __hip_texture This avoids the use of extra casts when obtaining a texture object handle. Change-Id: I42df22bdad0ab9ac6c33cb8b282dee65fe7cfd6e --- vdi/hip_texture.cpp | 74 +++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index 19761d1e78..f838d6cb0c 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -24,36 +24,34 @@ #include "hip_conversions.hpp" #include "platform/sampler.hpp" -namespace hip { - struct TextureObject { - uint32_t imageSRD[HIP_IMAGE_OBJECT_SIZE_DWORD]; - uint32_t samplerSRD[HIP_SAMPLER_OBJECT_SIZE_DWORD]; - amd::Image* image; - amd::Sampler* sampler; - hipResourceDesc resDesc; - hipTextureDesc texDesc; - hipResourceViewDesc resViewDesc; +struct __hip_texture { + uint32_t imageSRD[HIP_IMAGE_OBJECT_SIZE_DWORD]; + uint32_t samplerSRD[HIP_SAMPLER_OBJECT_SIZE_DWORD]; + amd::Image* image; + amd::Sampler* sampler; + hipResourceDesc resDesc; + hipTextureDesc texDesc; + hipResourceViewDesc resViewDesc; - TextureObject(amd::Image* image_, - amd::Sampler* sampler_, - const hipResourceDesc& resDesc_, - const hipTextureDesc& texDesc_, - const hipResourceViewDesc& resViewDesc_) : - image(image_), - sampler(sampler_), - resDesc(resDesc_), - texDesc(texDesc_), - resViewDesc(resViewDesc_) { - amd::Context& context = *hip::getCurrentDevice()->asContext(); - amd::Device& device = *context.devices()[0]; + __hip_texture(amd::Image* image_, + amd::Sampler* sampler_, + const hipResourceDesc& resDesc_, + const hipTextureDesc& texDesc_, + const hipResourceViewDesc& resViewDesc_) : + image(image_), + sampler(sampler_), + resDesc(resDesc_), + texDesc(texDesc_), + resViewDesc(resViewDesc_) { + amd::Context& context = *hip::getCurrentDevice()->asContext(); + amd::Device& device = *context.devices()[0]; - device::Memory* imageMem = image->getDeviceMemory(device); - std::memcpy(imageSRD, imageMem->cpuSrd(), sizeof(imageSRD)); + device::Memory* imageMem = image->getDeviceMemory(device); + std::memcpy(imageSRD, imageMem->cpuSrd(), sizeof(imageSRD)); - device::Sampler* samplerMem = sampler->getDeviceSampler(device); - std::memcpy(samplerSRD, samplerMem->hwState(), sizeof(samplerSRD)); - } - }; + device::Sampler* samplerMem = sampler->getDeviceSampler(device); + std::memcpy(samplerSRD, samplerMem->hwState(), sizeof(samplerSRD)); + } }; amd::Image* ihipImageCreate(const cl_channel_order channelOrder, @@ -290,11 +288,11 @@ hipError_t ihipCreateTextureObject(hipTextureObject_t* pTexObject, } void *texObjectBuffer = nullptr; - ihipMalloc(&texObjectBuffer, sizeof(hip::TextureObject), CL_MEM_SVM_FINE_GRAIN_BUFFER); + ihipMalloc(&texObjectBuffer, sizeof(__hip_texture), CL_MEM_SVM_FINE_GRAIN_BUFFER); if (texObjectBuffer == nullptr) { return hipErrorOutOfMemory; } - *pTexObject = reinterpret_cast(new (texObjectBuffer) hip::TextureObject{image, sampler, *pResDesc, *pTexDesc, (pResViewDesc != nullptr) ? *pResViewDesc : hipResourceViewDesc{}}); + *pTexObject = new (texObjectBuffer) __hip_texture{image, sampler, *pResDesc, *pTexDesc, (pResViewDesc != nullptr) ? *pResViewDesc : hipResourceViewDesc{}}; return hipSuccess; } @@ -314,17 +312,16 @@ hipError_t ihipDestroyTextureObject(hipTextureObject_t texObject) { return hipErrorInvalidValue; } - hip::TextureObject* hipTexObject = reinterpret_cast(texObject); - const hipResourceType type = hipTexObject->resDesc.resType; + const hipResourceType type = texObject->resDesc.resType; const bool isImageFromBuffer = (type == hipResourceTypeLinear) || (type == hipResourceTypePitch2D); const bool isImageView = ((type == hipResourceTypeArray) || (type == hipResourceTypeMipmappedArray)) && - !hipTexObject->image->isParent(); + !texObject->image->isParent(); if (isImageFromBuffer || isImageView) { - hipTexObject->image->release(); + texObject->image->release(); } // TODO Should call ihipFree() to not polute the api trace. - return hipFree(hipTexObject); + return hipFree(texObject); } hipError_t hipDestroyTextureObject(hipTextureObject_t texObject) { @@ -343,8 +340,7 @@ hipError_t hipGetTextureObjectResourceDesc(hipResourceDesc* pResDesc, HIP_RETURN(hipErrorInvalidValue); } - hip::TextureObject* hipTexObject = reinterpret_cast(texObject); - *pResDesc = hipTexObject->resDesc; + *pResDesc = texObject->resDesc; HIP_RETURN(hipSuccess); } @@ -358,8 +354,7 @@ hipError_t hipGetTextureObjectResourceViewDesc(hipResourceViewDesc* pResViewDesc HIP_RETURN(hipErrorInvalidValue); } - hip::TextureObject* hipTexObject = reinterpret_cast(texObject); - *pResViewDesc = hipTexObject->resViewDesc; + *pResViewDesc = texObject->resViewDesc; HIP_RETURN(hipSuccess); } @@ -373,8 +368,7 @@ hipError_t hipGetTextureObjectTextureDesc(hipTextureDesc* pTexDesc, HIP_RETURN(hipErrorInvalidValue); } - hip::TextureObject* hipTexObject = reinterpret_cast(texObject); - *pTexDesc = hipTexObject->texDesc; + *pTexDesc = texObject->texDesc; HIP_RETURN(hipSuccess); } From faf3b83594603bc407fabdd0ec37495cb6526321 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 20 Mar 2020 18:45:48 -0400 Subject: [PATCH 132/154] Handle offsets for dptr <-> image copies Change-Id: I7a4a56ee07a26a741d2aac35502446d248f720ad --- vdi/hip_memory.cpp | 48 +++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index b38c2dbdbc..f197ba2bef 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -886,15 +886,11 @@ hipError_t ihipMemcpyAtoD(hipArray* srcArray, void* dstDevice, amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D region, + amd::Coord3D copyRegion, size_t dstRowPitch, size_t dstSlicePitch, hipStream_t stream, bool isAsync = false) { - // TODO VDI doesn't support 2D/3D image to buffer copy. - (void)dstRowPitch; - (void)dstSlicePitch; - cl_mem srcMemObj = reinterpret_cast(srcArray->data); if (is_valid(srcMemObj) == false) { return hipErrorInvalidValue; @@ -905,7 +901,19 @@ hipError_t ihipMemcpyAtoD(hipArray* srcArray, amd::Memory* dstMemory = getMemoryObject(dstDevice, offset); assert(offset != 0); - if (!srcImage->validateRegion(srcOrigin, region)) { + amd::BufferRect srcRect; + if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcImage->getRowPitch(), srcImage->getSlicePitch())) { + return hipErrorInvalidValue; + } + + amd::BufferRect dstRect; + if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), dstRowPitch, dstSlicePitch)) { + return hipErrorInvalidValue; + } + + const size_t copySizeInBytes = copyRegion[0] * copyRegion[1] * copyRegion[2] * srcImage->getImageFormat().getElementSize(); + if (!srcImage->validateRegion(srcOrigin, copyRegion) || + !dstMemory->validateRegion(dstOrigin, {copySizeInBytes, 0, 0})) { return hipErrorInvalidValue; } @@ -916,7 +924,9 @@ hipError_t ihipMemcpyAtoD(hipArray* srcArray, *dstMemory, srcOrigin, dstOrigin, - region); + copyRegion, + srcRect, + dstRect); if (command == nullptr) { return hipErrorOutOfMemory; @@ -935,15 +945,11 @@ hipError_t ihipMemcpyDtoA(void* srcDevice, hipArray* dstArray, amd::Coord3D srcOrigin, amd::Coord3D dstOrigin, - amd::Coord3D region, + amd::Coord3D copyRegion, size_t srcRowPitch, size_t srcSlicePitch, hipStream_t stream, bool isAsync = false) { - // TODO VDI doesn't support 2D/3D buffer to image copy. - (void)srcRowPitch; - (void)srcSlicePitch; - cl_mem dstMemObj = reinterpret_cast(dstArray->data); if (is_valid(dstMemObj) == false) { return hipErrorInvalidValue; @@ -954,9 +960,19 @@ hipError_t ihipMemcpyDtoA(void* srcDevice, assert(offset != 0); amd::Image* dstImage = as_amd(dstMemObj)->asImage(); - const size_t copySizeInBytes = region[0] * region[1] * region[2] * dstImage->getImageFormat().getElementSize(); + amd::BufferRect srcRect; + if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { + return hipErrorInvalidValue; + } + + amd::BufferRect dstRect; + if (!dstRect.create(static_cast(dstOrigin), static_cast(copyRegion), dstImage->getRowPitch(), dstImage->getSlicePitch())) { + return hipErrorInvalidValue; + } + + const size_t copySizeInBytes = copyRegion[0] * copyRegion[1] * copyRegion[2] * dstImage->getImageFormat().getElementSize(); if (!srcMemory->validateRegion(srcOrigin, {copySizeInBytes, 0, 0}) || - !dstImage->validateRegion(dstOrigin, region)) { + !dstImage->validateRegion(dstOrigin, copyRegion)) { return hipErrorInvalidValue; } @@ -967,7 +983,9 @@ hipError_t ihipMemcpyDtoA(void* srcDevice, *dstImage, srcOrigin, dstOrigin, - region); + copyRegion, + srcRect, + dstRect); if (command == nullptr) { return hipErrorOutOfMemory; From e0187ba405b116870f9bbbbfa9d5055ef557c782 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Mon, 23 Mar 2020 16:26:21 -0400 Subject: [PATCH 133/154] Add initial entry points for mipmapped array API Change-Id: Icd59cc7323ddcb6773da6105260415a1e6f4cdcb --- include/hip/hcc_detail/hip_runtime_api.h | 13 +++++++ vdi/hip_hcc.def.in | 6 ++++ vdi/hip_hcc.map.in | 6 ++++ vdi/hip_memory.cpp | 46 ++++++++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index d6b66090cd..d35d426a5e 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -3478,6 +3478,19 @@ hipError_t hipTexRefSetMipmappedArray( textureReference* texRef, hipMipmappedArray* mipmappedArray, unsigned int Flags); + +hipError_t hipMipmappedArrayCreate( + hipMipmappedArray_t* pHandle, + HIP_ARRAY3D_DESCRIPTOR* pMipmappedArrayDesc, + unsigned int numMipmapLevels); + +hipError_t hipMipmappedArrayDestroy( + hipMipmappedArray_t hMipmappedArray); + +hipError_t hipMipmappedArrayGetLevel( + hipArray_t* pLevelArray, + hipMipmappedArray_t hMipMappedArray, + unsigned int level); #endif /** diff --git a/vdi/hip_hcc.def.in b/vdi/hip_hcc.def.in index c6e4a00200..82d0283e1a 100644 --- a/vdi/hip_hcc.def.in +++ b/vdi/hip_hcc.def.in @@ -231,3 +231,9 @@ hiprtcGetProgramLogSize hiprtcGetCode hiprtcGetCodeSize hiprtcGetErrorString +hipMipmappedArrayCreate +hipMallocMipmappedArray +hipMipmappedArrayDestroy +hipFreeMipmappedArray +hipMipmappedArrayGetLevel +hipGetMipmappedArrayLevel diff --git a/vdi/hip_hcc.map.in b/vdi/hip_hcc.map.in index c26e79fc70..210789e420 100644 --- a/vdi/hip_hcc.map.in +++ b/vdi/hip_hcc.map.in @@ -225,6 +225,12 @@ global: hipTexRefSetMipmapLevelBias; hipTexRefSetMipmapLevelClamp; hipTexRefSetMipmappedArray; + hipMipmappedArrayCreate; + hipMallocMipmappedArray; + hipMipmappedArrayDestroy; + hipFreeMipmappedArray; + hipMipmappedArrayGetLevel; + hipGetMipmappedArrayLevel; extern "C++" { hip_impl::hipLaunchKernelGGLImpl*; hip_impl::demangle*; diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index fdcfa78ebc..718d907c9e 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -2055,3 +2055,49 @@ hipError_t hipMemcpyHtoAAsync(hipArray* dstArray, HIP_RETURN(ihipMemcpyHtoA(srcHost, dstArray, {0, 0, 0}, {dstOffset, 0, 0}, {ByteCount, 1, 1}, 0, 0, stream, true)); } + +hipError_t hipMipmappedArrayCreate(hipMipmappedArray_t* pHandle, + HIP_ARRAY3D_DESCRIPTOR* pMipmappedArrayDesc, + unsigned int numMipmapLevels) { + HIP_INIT_API(hipMipmappedArrayCreate, pHandle, pMipmappedArrayDesc, numMipmapLevels); + + HIP_RETURN(hipErrorNotSupported); +} + +hipError_t hipMallocMipmappedArray(hipMipmappedArray_t *mipmappedArray, + const hipChannelFormatDesc* desc, + hipExtent extent, + unsigned int numLevels, + unsigned int flags) { + HIP_INIT_API(hipMallocMipmappedArray, mipmappedArray, desc, &extent, numLevels, flags); + + HIP_RETURN(hipErrorNotSupported); +} + +hipError_t hipMipmappedArrayDestroy(hipMipmappedArray_t hMipmappedArray) { + HIP_INIT_API(hipMipmappedArrayDestroy, hMipmappedArray); + + HIP_RETURN(hipErrorNotSupported); +} + +hipError_t hipFreeMipmappedArray(hipMipmappedArray_t mipmappedArray) { + HIP_INIT_API(hipFreeMipmappedArray, mipmappedArray); + + HIP_RETURN(hipErrorNotSupported); +} + +hipError_t hipMipmappedArrayGetLevel(hipArray_t* pLevelArray, + hipMipmappedArray_t hMipMappedArray, + unsigned int level) { + HIP_INIT_API(hipMipmappedArrayGetLevel, pLevelArray, hMipMappedArray, level); + + HIP_RETURN(hipErrorNotSupported); +} + +hipError_t hipGetMipmappedArrayLevel(hipArray_t *levelArray, + hipMipmappedArray_const_t mipmappedArray, + unsigned int level) { + HIP_INIT_API(hipGetMipmappedArrayLevel, levelArray, mipmappedArray, level); + + HIP_RETURN(hipErrorNotSupported); +} From 8ddeeb45515ef7cc5e4c66b50aed4084a3f0ffac Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Mon, 23 Mar 2020 14:11:25 -0400 Subject: [PATCH 134/154] Enable initial sRGB support Instead of using the sampler field force_degamma to perform sRGB->linear conversion during pixel sampling, we use an appropriate image format instead. The overhead of this is having to create an image view when creating a texture object from an array. Change-Id: I1ca368c312c1fd4b6f784a3a1b35b5eeb28070ff --- vdi/hip_conversions.hpp | 7 +++++-- vdi/hip_memory.cpp | 2 +- vdi/hip_texture.cpp | 15 ++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index 3b43cae2b1..0b8557ee4d 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -76,14 +76,17 @@ cl_channel_type getCLChannelType(const hipArray_Format hipFormat, } inline -cl_channel_order getCLChannelOrder(const unsigned int hipNumChannels) { +cl_channel_order getCLChannelOrder(const unsigned int hipNumChannels, + const int sRGB) { switch (hipNumChannels) { case 1: return CL_R; case 2: return CL_RG; case 4: - return CL_RGBA; + return (sRGB == 1) ? CL_sRGBA : CL_RGBA; + default: + break; } ShouldNotReachHere(); diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index f197ba2bef..fdcfa78ebc 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -545,7 +545,7 @@ hipError_t ihipArrayCreate(hipArray** array, return hipErrorNotSupported; } - const cl_channel_order channelOrder = hip::getCLChannelOrder(pAllocateArray->NumChannels); + const cl_channel_order channelOrder = hip::getCLChannelOrder(pAllocateArray->NumChannels, 0); const cl_channel_type channelType = hip::getCLChannelType(pAllocateArray->Format, hipReadModeElementType); const cl_mem_object_type imageType = hip::getCLMemObjectType(pAllocateArray->Width, pAllocateArray->Height, diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index b2e58a7f69..6b2f151922 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -130,10 +130,6 @@ hipError_t ihipCreateTextureObject(hipTextureObject_t* pTexObject, if (pTexDesc->addressMode[0] == hipAddressModeBorder) { return hipErrorNotSupported; } - // We don't program the force_degamma/skip_degamma fields in the HW sampler SRD. - if (pTexDesc->sRGB == 1) { - return hipErrorNotSupported; - } // We don't program the max_ansio_ratio field in the the HW sampler SRD. if (pTexDesc->maxAnisotropy != 0) { return hipErrorNotSupported; @@ -221,10 +217,11 @@ hipError_t ihipCreateTextureObject(hipTextureObject_t* pTexObject, // We need to create an image view if the user requested to use normalized pixel values, // due to already having the image created with a different format. if ((pResViewDesc != nullptr) || - (readMode == hipReadModeNormalizedFloat)) { + (readMode == hipReadModeNormalizedFloat) || + (pTexDesc->sRGB == 1)) { // TODO VDI currently right now can only change the format of the image. - const cl_channel_order channelOrder = (pResViewDesc != nullptr) ? hip::getCLChannelOrder(hip::getNumChannels(pResViewDesc->format)) : - hip::getCLChannelOrder(pResDesc->res.array.array->NumChannels); + const cl_channel_order channelOrder = (pResViewDesc != nullptr) ? hip::getCLChannelOrder(hip::getNumChannels(pResViewDesc->format), pTexDesc->sRGB) : + hip::getCLChannelOrder(pResDesc->res.array.array->NumChannels, pTexDesc->sRGB); const cl_channel_type channelType = (pResViewDesc != nullptr) ? hip::getCLChannelType(hip::getArrayFormat(pResViewDesc->format), readMode) : hip::getCLChannelType(pResDesc->res.array.array->Format, readMode); const amd::Image::Format imageFormat(cl_image_format{channelOrder, channelType}); @@ -244,7 +241,7 @@ hipError_t ihipCreateTextureObject(hipTextureObject_t* pTexObject, break; } case hipResourceTypeLinear: { - const cl_channel_order channelOrder = hip::getCLChannelOrder(hip::getNumChannels(pResDesc->res.linear.desc)); + const cl_channel_order channelOrder = hip::getCLChannelOrder(hip::getNumChannels(pResDesc->res.linear.desc), pTexDesc->sRGB); const cl_channel_type channelType = hip::getCLChannelType(hip::getArrayFormat(pResDesc->res.linear.desc), pTexDesc->readMode); const amd::Image::Format imageFormat({channelOrder, channelType}); const cl_mem_object_type imageType = hip::getCLMemObjectType(pResDesc->resType); @@ -268,7 +265,7 @@ hipError_t ihipCreateTextureObject(hipTextureObject_t* pTexObject, break; } case hipResourceTypePitch2D: { - const cl_channel_order channelOrder = hip::getCLChannelOrder(hip::getNumChannels(pResDesc->res.pitch2D.desc)); + const cl_channel_order channelOrder = hip::getCLChannelOrder(hip::getNumChannels(pResDesc->res.pitch2D.desc), pTexDesc->sRGB); const cl_channel_type channelType = hip::getCLChannelType(hip::getArrayFormat(pResDesc->res.pitch2D.desc), pTexDesc->readMode); const cl_mem_object_type imageType = hip::getCLMemObjectType(pResDesc->resType); size_t offset = 0; From 2a2b9e47a0a16020e83722d6201d45b8e333d630 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Mon, 23 Mar 2020 16:31:30 -0400 Subject: [PATCH 135/154] Add support for formating hipExtent objects Change-Id: Iea54a510e81a856c0c450305b3e5a7179ee48295 --- vdi/hip_formatting.hpp | 11 +++++++++++ vdi/hip_memory.cpp | 11 +++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/vdi/hip_formatting.hpp b/vdi/hip_formatting.hpp index ec92b137c1..8c26249e03 100644 --- a/vdi/hip_formatting.hpp +++ b/vdi/hip_formatting.hpp @@ -829,4 +829,15 @@ inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY3D_DESCRIPTOR* os << "nullptr"; } return os; +} + +inline std::ostream& operator<<(std::ostream& os, const hipExtent& s) { + os << '{' + << s.width + << ',' + << s.height + << ',' + << s.depth + << '}'; + return os; } \ No newline at end of file diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 718d907c9e..2ffa356907 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -395,7 +395,7 @@ hipError_t hipMallocPitch(void** ptr, size_t* pitch, size_t width, size_t height } hipError_t hipMalloc3D(hipPitchedPtr* pitchedDevPtr, hipExtent extent) { - HIP_INIT_API(hipMalloc3D, pitchedDevPtr, &extent); + HIP_INIT_API(hipMalloc3D, pitchedDevPtr, extent); size_t pitch = 0; @@ -628,8 +628,7 @@ hipError_t hipMalloc3DArray(hipArray_t* array, const hipChannelFormatDesc* desc, hipExtent extent, unsigned int flags) { - // TODO overload operator<<(ostream&, hipExtent&). - HIP_INIT_API(hipMalloc3DArray, array, desc, &extent, flags); + HIP_INIT_API(hipMalloc3DArray, array, desc, extent, flags); HIP_ARRAY3D_DESCRIPTOR allocateArray = {extent.width, extent.height, @@ -1724,7 +1723,7 @@ hipError_t hipMemset2DAsync(void* dst, size_t pitch, int value, } hipError_t hipMemset3D(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent) { - HIP_INIT_API(hipMemset3D, pitchedDevPtr, value, &extent); + HIP_INIT_API(hipMemset3D, pitchedDevPtr, value, extent); void *dst = pitchedDevPtr.ptr; size_t sizeBytes = pitchedDevPtr.pitch * extent.height * extent.depth; @@ -1733,7 +1732,7 @@ hipError_t hipMemset3D(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent) } hipError_t hipMemset3DAsync(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent, hipStream_t stream) { - HIP_INIT_API(hipMemset3DAsync, pitchedDevPtr, value, &extent, stream); + HIP_INIT_API(hipMemset3DAsync, pitchedDevPtr, value, extent, stream); void *dst = pitchedDevPtr.ptr; size_t sizeBytes = pitchedDevPtr.pitch * extent.height * extent.depth; @@ -2069,7 +2068,7 @@ hipError_t hipMallocMipmappedArray(hipMipmappedArray_t *mipmappedArray, hipExtent extent, unsigned int numLevels, unsigned int flags) { - HIP_INIT_API(hipMallocMipmappedArray, mipmappedArray, desc, &extent, numLevels, flags); + HIP_INIT_API(hipMallocMipmappedArray, mipmappedArray, desc, extent, numLevels, flags); HIP_RETURN(hipErrorNotSupported); } From a5fb06f4db4fa3c62680e3613e416b9f9ccf469d Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Thu, 26 Mar 2020 17:35:09 -0700 Subject: [PATCH 136/154] OpenCL2.2 Header changes Change-Id: I3285a8ffc7e34333feb68e925ebd460831c8e3d4 --- amdocl/CL/cl.h | 1639 ++++++++++++++++++------------ amdocl/CL/cl_egl.h | 94 +- amdocl/CL/cl_ext.h | 696 ++++++++----- amdocl/CL/cl_gl.h | 140 +-- amdocl/CL/cl_gl_ext.h | 34 +- amdocl/CL/cl_icd.h | 1269 +++++++++++++++++++++++ amdocl/CL/cl_platform.h | 402 +++----- amdocl/CL/cl_version.h | 86 ++ amdocl/CL/opencl.h | 12 - amdocl/cl_icd.cpp | 293 ++++++ amdocl/icd/loader/icd_dispatch.h | 1423 +------------------------- vdi/CMakeLists.txt | 2 +- vdi/fixme.cpp | 2 +- 13 files changed, 3397 insertions(+), 2695 deletions(-) create mode 100644 amdocl/CL/cl_icd.h create mode 100644 amdocl/CL/cl_version.h create mode 100644 amdocl/cl_icd.cpp diff --git a/amdocl/CL/cl.h b/amdocl/CL/cl.h index 43ac8ab1ea..cea6dc2405 100644 --- a/amdocl/CL/cl.h +++ b/amdocl/CL/cl.h @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2008-2015 The Khronos Group Inc. + * Copyright (c) 2008-2019 The Khronos Group Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and/or associated documentation files (the @@ -29,11 +29,8 @@ #ifndef __OPENCL_CL_H #define __OPENCL_CL_H -#ifdef __APPLE__ -#include -#else +#include #include -#endif #ifdef __cplusplus extern "C" { @@ -51,7 +48,7 @@ typedef struct _cl_kernel * cl_kernel; typedef struct _cl_event * cl_event; typedef struct _cl_sampler * cl_sampler; -typedef cl_uint cl_bool; /* WARNING! Unlike cl_ types in cl_platform.h, cl_bool is not guaranteed to be the same size as the bool in kernels. */ +typedef cl_uint cl_bool; /* WARNING! Unlike cl_ types in cl_platform.h, cl_bool is not guaranteed to be the same size as the bool in kernels. */ typedef cl_ulong cl_bitfield; typedef cl_bitfield cl_device_type; typedef cl_uint cl_platform_info; @@ -60,52 +57,80 @@ typedef cl_bitfield cl_device_fp_config; typedef cl_uint cl_device_mem_cache_type; typedef cl_uint cl_device_local_mem_type; typedef cl_bitfield cl_device_exec_capabilities; +#ifdef CL_VERSION_2_0 typedef cl_bitfield cl_device_svm_capabilities; +#endif typedef cl_bitfield cl_command_queue_properties; +#ifdef CL_VERSION_1_2 typedef intptr_t cl_device_partition_property; typedef cl_bitfield cl_device_affinity_domain; +#endif typedef intptr_t cl_context_properties; typedef cl_uint cl_context_info; +#ifdef CL_VERSION_2_0 typedef cl_bitfield cl_queue_properties; +#endif typedef cl_uint cl_command_queue_info; typedef cl_uint cl_channel_order; typedef cl_uint cl_channel_type; typedef cl_bitfield cl_mem_flags; +#ifdef CL_VERSION_2_0 typedef cl_bitfield cl_svm_mem_flags; +#endif typedef cl_uint cl_mem_object_type; typedef cl_uint cl_mem_info; +#ifdef CL_VERSION_1_2 typedef cl_bitfield cl_mem_migration_flags; +#endif typedef cl_uint cl_image_info; +#ifdef CL_VERSION_1_1 typedef cl_uint cl_buffer_create_type; +#endif typedef cl_uint cl_addressing_mode; typedef cl_uint cl_filter_mode; typedef cl_uint cl_sampler_info; typedef cl_bitfield cl_map_flags; +#ifdef CL_VERSION_2_0 typedef intptr_t cl_pipe_properties; typedef cl_uint cl_pipe_info; +#endif typedef cl_uint cl_program_info; typedef cl_uint cl_program_build_info; +#ifdef CL_VERSION_1_2 typedef cl_uint cl_program_binary_type; +#endif typedef cl_int cl_build_status; typedef cl_uint cl_kernel_info; +#ifdef CL_VERSION_1_2 typedef cl_uint cl_kernel_arg_info; typedef cl_uint cl_kernel_arg_address_qualifier; typedef cl_uint cl_kernel_arg_access_qualifier; typedef cl_bitfield cl_kernel_arg_type_qualifier; +#endif typedef cl_uint cl_kernel_work_group_info; +#ifdef CL_VERSION_2_1 typedef cl_uint cl_kernel_sub_group_info; +#endif typedef cl_uint cl_event_info; typedef cl_uint cl_command_type; typedef cl_uint cl_profiling_info; +#ifdef CL_VERSION_2_0 typedef cl_bitfield cl_sampler_properties; typedef cl_uint cl_kernel_exec_info; +#endif +#ifdef CL_EXPERIMENTAL +typedef cl_bitfield cl_device_atomic_capabilities; +typedef cl_uint cl_khronos_vendor_id; +#endif typedef struct _cl_image_format { cl_channel_order image_channel_order; cl_channel_type image_channel_data_type; } cl_image_format; +#ifdef CL_VERSION_1_2 + typedef struct _cl_image_desc { cl_mem_object_type image_type; size_t image_width; @@ -116,20 +141,36 @@ typedef struct _cl_image_desc { size_t image_slice_pitch; cl_uint num_mip_levels; cl_uint num_samples; +#ifdef CL_VERSION_2_0 #ifdef __GNUC__ __extension__ /* Prevents warnings about anonymous union in -pedantic builds */ +#endif +#ifdef _MSC_VER +#pragma warning( push ) +#pragma warning( disable : 4201 ) /* Prevents warning about nameless struct/union in /W4 /Za builds */ #endif union { +#endif cl_mem buffer; +#ifdef CL_VERSION_2_0 cl_mem mem_object; }; +#ifdef _MSC_VER +#pragma warning( pop ) +#endif +#endif } cl_image_desc; +#endif + +#ifdef CL_VERSION_1_1 + typedef struct _cl_buffer_region { size_t origin; size_t size; } cl_buffer_region; +#endif /******************************************************************************/ @@ -147,13 +188,17 @@ typedef struct _cl_buffer_region { #define CL_IMAGE_FORMAT_NOT_SUPPORTED -10 #define CL_BUILD_PROGRAM_FAILURE -11 #define CL_MAP_FAILURE -12 +#ifdef CL_VERSION_1_1 #define CL_MISALIGNED_SUB_BUFFER_OFFSET -13 #define CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST -14 +#endif +#ifdef CL_VERSION_1_2 #define CL_COMPILE_PROGRAM_FAILURE -15 #define CL_LINKER_NOT_AVAILABLE -16 #define CL_LINK_PROGRAM_FAILURE -17 #define CL_DEVICE_PARTITION_FAILED -18 #define CL_KERNEL_ARG_INFO_NOT_AVAILABLE -19 +#endif #define CL_INVALID_VALUE -30 #define CL_INVALID_DEVICE_TYPE -31 @@ -189,29 +234,32 @@ typedef struct _cl_buffer_region { #define CL_INVALID_BUFFER_SIZE -61 #define CL_INVALID_MIP_LEVEL -62 #define CL_INVALID_GLOBAL_WORK_SIZE -63 +#ifdef CL_VERSION_1_1 #define CL_INVALID_PROPERTY -64 +#endif +#ifdef CL_VERSION_1_2 #define CL_INVALID_IMAGE_DESCRIPTOR -65 #define CL_INVALID_COMPILER_OPTIONS -66 #define CL_INVALID_LINKER_OPTIONS -67 #define CL_INVALID_DEVICE_PARTITION_COUNT -68 +#endif +#ifdef CL_VERSION_2_0 #define CL_INVALID_PIPE_SIZE -69 #define CL_INVALID_DEVICE_QUEUE -70 +#endif +#ifdef CL_VERSION_2_2 #define CL_INVALID_SPEC_ID -71 #define CL_MAX_SIZE_RESTRICTION_EXCEEDED -72 +#endif -/* OpenCL Version */ -#define CL_VERSION_1_0 1 -#define CL_VERSION_1_1 1 -#define CL_VERSION_1_2 1 -#define CL_VERSION_2_0 1 -#define CL_VERSION_2_1 1 -#define CL_VERSION_2_2 1 /* cl_bool */ #define CL_FALSE 0 #define CL_TRUE 1 +#ifdef CL_VERSION_1_2 #define CL_BLOCKING CL_TRUE #define CL_NON_BLOCKING CL_FALSE +#endif /* cl_platform_info */ #define CL_PLATFORM_PROFILE 0x0900 @@ -219,14 +267,18 @@ typedef struct _cl_buffer_region { #define CL_PLATFORM_NAME 0x0902 #define CL_PLATFORM_VENDOR 0x0903 #define CL_PLATFORM_EXTENSIONS 0x0904 +#ifdef CL_VERSION_2_1 #define CL_PLATFORM_HOST_TIMER_RESOLUTION 0x0905 +#endif /* cl_device_type - bitfield */ #define CL_DEVICE_TYPE_DEFAULT (1 << 0) #define CL_DEVICE_TYPE_CPU (1 << 1) #define CL_DEVICE_TYPE_GPU (1 << 2) #define CL_DEVICE_TYPE_ACCELERATOR (1 << 3) +#ifdef CL_VERSION_1_2 #define CL_DEVICE_TYPE_CUSTOM (1 << 4) +#endif #define CL_DEVICE_TYPE_ALL 0xFFFFFFFF /* cl_device_info */ @@ -273,7 +325,9 @@ typedef struct _cl_buffer_region { #define CL_DEVICE_COMPILER_AVAILABLE 0x1028 #define CL_DEVICE_EXECUTION_CAPABILITIES 0x1029 #define CL_DEVICE_QUEUE_PROPERTIES 0x102A /* deprecated */ +#ifdef CL_VERSION_2_0 #define CL_DEVICE_QUEUE_ON_HOST_PROPERTIES 0x102A +#endif #define CL_DEVICE_NAME 0x102B #define CL_DEVICE_VENDOR 0x102C #define CL_DRIVER_VERSION 0x102D @@ -281,8 +335,11 @@ typedef struct _cl_buffer_region { #define CL_DEVICE_VERSION 0x102F #define CL_DEVICE_EXTENSIONS 0x1030 #define CL_DEVICE_PLATFORM 0x1031 +#ifdef CL_VERSION_1_2 #define CL_DEVICE_DOUBLE_FP_CONFIG 0x1032 -#define CL_DEVICE_HALF_FP_CONFIG 0x1033 +#endif +/* 0x1033 reserved for CL_DEVICE_HALF_FP_CONFIG which is already defined in "cl_ext.h" */ +#ifdef CL_VERSION_1_1 #define CL_DEVICE_PREFERRED_VECTOR_WIDTH_HALF 0x1034 #define CL_DEVICE_HOST_UNIFIED_MEMORY 0x1035 /* deprecated */ #define CL_DEVICE_NATIVE_VECTOR_WIDTH_CHAR 0x1036 @@ -293,6 +350,8 @@ typedef struct _cl_buffer_region { #define CL_DEVICE_NATIVE_VECTOR_WIDTH_DOUBLE 0x103B #define CL_DEVICE_NATIVE_VECTOR_WIDTH_HALF 0x103C #define CL_DEVICE_OPENCL_C_VERSION 0x103D +#endif +#ifdef CL_VERSION_1_2 #define CL_DEVICE_LINKER_AVAILABLE 0x103E #define CL_DEVICE_BUILT_IN_KERNELS 0x103F #define CL_DEVICE_IMAGE_MAX_BUFFER_SIZE 0x1040 @@ -305,6 +364,8 @@ typedef struct _cl_buffer_region { #define CL_DEVICE_REFERENCE_COUNT 0x1047 #define CL_DEVICE_PREFERRED_INTEROP_USER_SYNC 0x1048 #define CL_DEVICE_PRINTF_BUFFER_SIZE 0x1049 +#endif +#ifdef CL_VERSION_2_0 #define CL_DEVICE_IMAGE_PITCH_ALIGNMENT 0x104A #define CL_DEVICE_IMAGE_BASE_ADDRESS_ALIGNMENT 0x104B #define CL_DEVICE_MAX_READ_WRITE_IMAGE_ARGS 0x104C @@ -322,9 +383,12 @@ typedef struct _cl_buffer_region { #define CL_DEVICE_PREFERRED_PLATFORM_ATOMIC_ALIGNMENT 0x1058 #define CL_DEVICE_PREFERRED_GLOBAL_ATOMIC_ALIGNMENT 0x1059 #define CL_DEVICE_PREFERRED_LOCAL_ATOMIC_ALIGNMENT 0x105A +#endif +#ifdef CL_VERSION_2_1 #define CL_DEVICE_IL_VERSION 0x105B #define CL_DEVICE_MAX_NUM_SUB_GROUPS 0x105C #define CL_DEVICE_SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS 0x105D +#endif /* cl_device_fp_config - bitfield */ #define CL_FP_DENORM (1 << 0) @@ -333,8 +397,12 @@ typedef struct _cl_buffer_region { #define CL_FP_ROUND_TO_ZERO (1 << 3) #define CL_FP_ROUND_TO_INF (1 << 4) #define CL_FP_FMA (1 << 5) +#ifdef CL_VERSION_1_1 #define CL_FP_SOFT_FLOAT (1 << 6) +#endif +#ifdef CL_VERSION_1_2 #define CL_FP_CORRECTLY_ROUNDED_DIVIDE_SQRT (1 << 7) +#endif /* cl_device_mem_cache_type */ #define CL_NONE 0x0 @@ -352,25 +420,37 @@ typedef struct _cl_buffer_region { /* cl_command_queue_properties - bitfield */ #define CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE (1 << 0) #define CL_QUEUE_PROFILING_ENABLE (1 << 1) +#ifdef CL_VERSION_2_0 #define CL_QUEUE_ON_DEVICE (1 << 2) #define CL_QUEUE_ON_DEVICE_DEFAULT (1 << 3) +#endif -/* cl_context_info */ +/* cl_context_info */ #define CL_CONTEXT_REFERENCE_COUNT 0x1080 #define CL_CONTEXT_DEVICES 0x1081 #define CL_CONTEXT_PROPERTIES 0x1082 +#ifdef CL_VERSION_1_1 #define CL_CONTEXT_NUM_DEVICES 0x1083 +#endif /* cl_context_properties */ #define CL_CONTEXT_PLATFORM 0x1084 +#ifdef CL_VERSION_1_2 #define CL_CONTEXT_INTEROP_USER_SYNC 0x1085 - +#endif + +#ifdef CL_VERSION_1_2 + /* cl_device_partition_property */ #define CL_DEVICE_PARTITION_EQUALLY 0x1086 #define CL_DEVICE_PARTITION_BY_COUNTS 0x1087 #define CL_DEVICE_PARTITION_BY_COUNTS_LIST_END 0x0 #define CL_DEVICE_PARTITION_BY_AFFINITY_DOMAIN 0x1088 - + +#endif + +#ifdef CL_VERSION_1_2 + /* cl_device_affinity_domain */ #define CL_DEVICE_AFFINITY_DOMAIN_NUMA (1 << 0) #define CL_DEVICE_AFFINITY_DOMAIN_L4_CACHE (1 << 1) @@ -378,20 +458,30 @@ typedef struct _cl_buffer_region { #define CL_DEVICE_AFFINITY_DOMAIN_L2_CACHE (1 << 3) #define CL_DEVICE_AFFINITY_DOMAIN_L1_CACHE (1 << 4) #define CL_DEVICE_AFFINITY_DOMAIN_NEXT_PARTITIONABLE (1 << 5) - + +#endif + +#ifdef CL_VERSION_2_0 + /* cl_device_svm_capabilities */ #define CL_DEVICE_SVM_COARSE_GRAIN_BUFFER (1 << 0) #define CL_DEVICE_SVM_FINE_GRAIN_BUFFER (1 << 1) #define CL_DEVICE_SVM_FINE_GRAIN_SYSTEM (1 << 2) #define CL_DEVICE_SVM_ATOMICS (1 << 3) +#endif + /* cl_command_queue_info */ #define CL_QUEUE_CONTEXT 0x1090 #define CL_QUEUE_DEVICE 0x1091 #define CL_QUEUE_REFERENCE_COUNT 0x1092 #define CL_QUEUE_PROPERTIES 0x1093 +#ifdef CL_VERSION_2_0 #define CL_QUEUE_SIZE 0x1094 +#endif +#ifdef CL_VERSION_2_1 #define CL_QUEUE_DEVICE_DEFAULT 0x1095 +#endif /* cl_mem_flags and cl_svm_mem_flags - bitfield */ #define CL_MEM_READ_WRITE (1 << 0) @@ -401,17 +491,25 @@ typedef struct _cl_buffer_region { #define CL_MEM_ALLOC_HOST_PTR (1 << 4) #define CL_MEM_COPY_HOST_PTR (1 << 5) /* reserved (1 << 6) */ +#ifdef CL_VERSION_1_2 #define CL_MEM_HOST_WRITE_ONLY (1 << 7) #define CL_MEM_HOST_READ_ONLY (1 << 8) #define CL_MEM_HOST_NO_ACCESS (1 << 9) +#endif +#ifdef CL_VERSION_2_0 #define CL_MEM_SVM_FINE_GRAIN_BUFFER (1 << 10) /* used by cl_svm_mem_flags only */ #define CL_MEM_SVM_ATOMICS (1 << 11) /* used by cl_svm_mem_flags only */ #define CL_MEM_KERNEL_READ_AND_WRITE (1 << 12) +#endif + +#ifdef CL_VERSION_1_2 /* cl_mem_migration_flags - bitfield */ #define CL_MIGRATE_MEM_OBJECT_HOST (1 << 0) #define CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED (1 << 1) +#endif + /* cl_channel_order */ #define CL_R 0x10B0 #define CL_A 0x10B1 @@ -423,16 +521,22 @@ typedef struct _cl_buffer_region { #define CL_ARGB 0x10B7 #define CL_INTENSITY 0x10B8 #define CL_LUMINANCE 0x10B9 +#ifdef CL_VERSION_1_1 #define CL_Rx 0x10BA #define CL_RGx 0x10BB #define CL_RGBx 0x10BC +#endif +#ifdef CL_VERSION_1_2 #define CL_DEPTH 0x10BD #define CL_DEPTH_STENCIL 0x10BE +#endif +#ifdef CL_VERSION_2_0 #define CL_sRGB 0x10BF #define CL_sRGBx 0x10C0 #define CL_sRGBA 0x10C1 #define CL_sBGRA 0x10C2 #define CL_ABGR 0x10C3 +#endif /* cl_channel_type */ #define CL_SNORM_INT8 0x10D0 @@ -450,18 +554,26 @@ typedef struct _cl_buffer_region { #define CL_UNSIGNED_INT32 0x10DC #define CL_HALF_FLOAT 0x10DD #define CL_FLOAT 0x10DE +#ifdef CL_VERSION_1_2 #define CL_UNORM_INT24 0x10DF +#endif +#ifdef CL_VERSION_2_1 #define CL_UNORM_INT_101010_2 0x10E0 +#endif /* cl_mem_object_type */ #define CL_MEM_OBJECT_BUFFER 0x10F0 #define CL_MEM_OBJECT_IMAGE2D 0x10F1 #define CL_MEM_OBJECT_IMAGE3D 0x10F2 +#ifdef CL_VERSION_1_2 #define CL_MEM_OBJECT_IMAGE2D_ARRAY 0x10F3 #define CL_MEM_OBJECT_IMAGE1D 0x10F4 #define CL_MEM_OBJECT_IMAGE1D_ARRAY 0x10F5 #define CL_MEM_OBJECT_IMAGE1D_BUFFER 0x10F6 +#endif +#ifdef CL_VERSION_2_0 #define CL_MEM_OBJECT_PIPE 0x10F7 +#endif /* cl_mem_info */ #define CL_MEM_TYPE 0x1100 @@ -471,9 +583,13 @@ typedef struct _cl_buffer_region { #define CL_MEM_MAP_COUNT 0x1104 #define CL_MEM_REFERENCE_COUNT 0x1105 #define CL_MEM_CONTEXT 0x1106 +#ifdef CL_VERSION_1_1 #define CL_MEM_ASSOCIATED_MEMOBJECT 0x1107 #define CL_MEM_OFFSET 0x1108 +#endif +#ifdef CL_VERSION_2_0 #define CL_MEM_USES_SVM_POINTER 0x1109 +#endif /* cl_image_info */ #define CL_IMAGE_FORMAT 0x1110 @@ -483,21 +599,29 @@ typedef struct _cl_buffer_region { #define CL_IMAGE_WIDTH 0x1114 #define CL_IMAGE_HEIGHT 0x1115 #define CL_IMAGE_DEPTH 0x1116 +#ifdef CL_VERSION_1_2 #define CL_IMAGE_ARRAY_SIZE 0x1117 #define CL_IMAGE_BUFFER 0x1118 #define CL_IMAGE_NUM_MIP_LEVELS 0x1119 #define CL_IMAGE_NUM_SAMPLES 0x111A - +#endif + +#ifdef CL_VERSION_2_0 + /* cl_pipe_info */ #define CL_PIPE_PACKET_SIZE 0x1120 #define CL_PIPE_MAX_PACKETS 0x1121 +#endif + /* cl_addressing_mode */ #define CL_ADDRESS_NONE 0x1130 #define CL_ADDRESS_CLAMP_TO_EDGE 0x1131 #define CL_ADDRESS_CLAMP 0x1132 #define CL_ADDRESS_REPEAT 0x1133 +#ifdef CL_VERSION_1_1 #define CL_ADDRESS_MIRRORED_REPEAT 0x1134 +#endif /* cl_filter_mode */ #define CL_FILTER_NEAREST 0x1140 @@ -509,14 +633,21 @@ typedef struct _cl_buffer_region { #define CL_SAMPLER_NORMALIZED_COORDS 0x1152 #define CL_SAMPLER_ADDRESSING_MODE 0x1153 #define CL_SAMPLER_FILTER_MODE 0x1154 +#ifdef CL_VERSION_2_0 +/* These enumerants are for the cl_khr_mipmap_image extension. + They have since been added to cl_ext.h with an appropriate + KHR suffix, but are left here for backwards compatibility. */ #define CL_SAMPLER_MIP_FILTER_MODE 0x1155 #define CL_SAMPLER_LOD_MIN 0x1156 #define CL_SAMPLER_LOD_MAX 0x1157 +#endif /* cl_map_flags - bitfield */ #define CL_MAP_READ (1 << 0) #define CL_MAP_WRITE (1 << 1) +#ifdef CL_VERSION_1_2 #define CL_MAP_WRITE_INVALIDATE_REGION (1 << 2) +#endif /* cl_program_info */ #define CL_PROGRAM_REFERENCE_COUNT 0x1160 @@ -526,25 +657,39 @@ typedef struct _cl_buffer_region { #define CL_PROGRAM_SOURCE 0x1164 #define CL_PROGRAM_BINARY_SIZES 0x1165 #define CL_PROGRAM_BINARIES 0x1166 +#ifdef CL_VERSION_1_2 #define CL_PROGRAM_NUM_KERNELS 0x1167 #define CL_PROGRAM_KERNEL_NAMES 0x1168 +#endif +#ifdef CL_VERSION_2_1 #define CL_PROGRAM_IL 0x1169 +#endif +#ifdef CL_VERSION_2_2 #define CL_PROGRAM_SCOPE_GLOBAL_CTORS_PRESENT 0x116A #define CL_PROGRAM_SCOPE_GLOBAL_DTORS_PRESENT 0x116B +#endif /* cl_program_build_info */ #define CL_PROGRAM_BUILD_STATUS 0x1181 #define CL_PROGRAM_BUILD_OPTIONS 0x1182 #define CL_PROGRAM_BUILD_LOG 0x1183 +#ifdef CL_VERSION_1_2 #define CL_PROGRAM_BINARY_TYPE 0x1184 +#endif +#ifdef CL_VERSION_2_0 #define CL_PROGRAM_BUILD_GLOBAL_VARIABLE_TOTAL_SIZE 0x1185 - +#endif + +#ifdef CL_VERSION_1_2 + /* cl_program_binary_type */ #define CL_PROGRAM_BINARY_TYPE_NONE 0x0 #define CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT 0x1 #define CL_PROGRAM_BINARY_TYPE_LIBRARY 0x2 #define CL_PROGRAM_BINARY_TYPE_EXECUTABLE 0x4 +#endif + /* cl_build_status */ #define CL_BUILD_SUCCESS 0 #define CL_BUILD_NONE -1 @@ -557,9 +702,11 @@ typedef struct _cl_buffer_region { #define CL_KERNEL_REFERENCE_COUNT 0x1192 #define CL_KERNEL_CONTEXT 0x1193 #define CL_KERNEL_PROGRAM 0x1194 +#ifdef CL_VERSION_1_2 #define CL_KERNEL_ATTRIBUTES 0x1195 -#define CL_KERNEL_MAX_NUM_SUB_GROUPS 0x11B9 -#define CL_KERNEL_COMPILE_NUM_SUB_GROUPS 0x11BA +#endif + +#ifdef CL_VERSION_1_2 /* cl_kernel_arg_info */ #define CL_KERNEL_ARG_ADDRESS_QUALIFIER 0x1196 @@ -568,24 +715,40 @@ typedef struct _cl_buffer_region { #define CL_KERNEL_ARG_TYPE_QUALIFIER 0x1199 #define CL_KERNEL_ARG_NAME 0x119A +#endif + +#ifdef CL_VERSION_1_2 + /* cl_kernel_arg_address_qualifier */ #define CL_KERNEL_ARG_ADDRESS_GLOBAL 0x119B #define CL_KERNEL_ARG_ADDRESS_LOCAL 0x119C #define CL_KERNEL_ARG_ADDRESS_CONSTANT 0x119D #define CL_KERNEL_ARG_ADDRESS_PRIVATE 0x119E +#endif + +#ifdef CL_VERSION_1_2 + /* cl_kernel_arg_access_qualifier */ #define CL_KERNEL_ARG_ACCESS_READ_ONLY 0x11A0 #define CL_KERNEL_ARG_ACCESS_WRITE_ONLY 0x11A1 #define CL_KERNEL_ARG_ACCESS_READ_WRITE 0x11A2 #define CL_KERNEL_ARG_ACCESS_NONE 0x11A3 - + +#endif + +#ifdef CL_VERSION_1_2 + /* cl_kernel_arg_type_qualifier */ #define CL_KERNEL_ARG_TYPE_NONE 0 #define CL_KERNEL_ARG_TYPE_CONST (1 << 0) #define CL_KERNEL_ARG_TYPE_RESTRICT (1 << 1) #define CL_KERNEL_ARG_TYPE_VOLATILE (1 << 2) +#ifdef CL_VERSION_2_0 #define CL_KERNEL_ARG_TYPE_PIPE (1 << 3) +#endif + +#endif /* cl_kernel_work_group_info */ #define CL_KERNEL_WORK_GROUP_SIZE 0x11B0 @@ -593,23 +756,37 @@ typedef struct _cl_buffer_region { #define CL_KERNEL_LOCAL_MEM_SIZE 0x11B2 #define CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE 0x11B3 #define CL_KERNEL_PRIVATE_MEM_SIZE 0x11B4 +#ifdef CL_VERSION_1_2 #define CL_KERNEL_GLOBAL_WORK_SIZE 0x11B5 +#endif + +#ifdef CL_VERSION_2_1 /* cl_kernel_sub_group_info */ #define CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE 0x2033 #define CL_KERNEL_SUB_GROUP_COUNT_FOR_NDRANGE 0x2034 #define CL_KERNEL_LOCAL_SIZE_FOR_SUB_GROUP_COUNT 0x11B8 - +#define CL_KERNEL_MAX_NUM_SUB_GROUPS 0x11B9 +#define CL_KERNEL_COMPILE_NUM_SUB_GROUPS 0x11BA + +#endif + +#ifdef CL_VERSION_2_0 + /* cl_kernel_exec_info */ #define CL_KERNEL_EXEC_INFO_SVM_PTRS 0x11B6 #define CL_KERNEL_EXEC_INFO_SVM_FINE_GRAIN_SYSTEM 0x11B7 -/* cl_event_info */ +#endif + +/* cl_event_info */ #define CL_EVENT_COMMAND_QUEUE 0x11D0 #define CL_EVENT_COMMAND_TYPE 0x11D1 #define CL_EVENT_REFERENCE_COUNT 0x11D2 #define CL_EVENT_COMMAND_EXECUTION_STATUS 0x11D3 +#ifdef CL_VERSION_1_1 #define CL_EVENT_CONTEXT 0x11D4 +#endif /* cl_command_type */ #define CL_COMMAND_NDRANGE_KERNEL 0x11F0 @@ -629,20 +806,25 @@ typedef struct _cl_buffer_region { #define CL_COMMAND_MARKER 0x11FE #define CL_COMMAND_ACQUIRE_GL_OBJECTS 0x11FF #define CL_COMMAND_RELEASE_GL_OBJECTS 0x1200 +#ifdef CL_VERSION_1_1 #define CL_COMMAND_READ_BUFFER_RECT 0x1201 #define CL_COMMAND_WRITE_BUFFER_RECT 0x1202 #define CL_COMMAND_COPY_BUFFER_RECT 0x1203 #define CL_COMMAND_USER 0x1204 +#endif +#ifdef CL_VERSION_1_2 #define CL_COMMAND_BARRIER 0x1205 #define CL_COMMAND_MIGRATE_MEM_OBJECTS 0x1206 #define CL_COMMAND_FILL_BUFFER 0x1207 #define CL_COMMAND_FILL_IMAGE 0x1208 +#endif +#ifdef CL_VERSION_2_0 #define CL_COMMAND_SVM_FREE 0x1209 #define CL_COMMAND_SVM_MEMCPY 0x120A #define CL_COMMAND_SVM_MEMFILL 0x120B #define CL_COMMAND_SVM_MAP 0x120C #define CL_COMMAND_SVM_UNMAP 0x120D -#define CL_COMMAND_SVM_MIGRATE_MEM 0x120E +#endif /* command execution status */ #define CL_COMPLETE 0x0 @@ -650,820 +832,1005 @@ typedef struct _cl_buffer_region { #define CL_SUBMITTED 0x2 #define CL_QUEUED 0x3 -/* cl_buffer_create_type */ +#ifdef CL_VERSION_1_1 + +/* cl_buffer_create_type */ #define CL_BUFFER_CREATE_TYPE_REGION 0x1220 -/* cl_profiling_info */ +#endif + +/* cl_profiling_info */ #define CL_PROFILING_COMMAND_QUEUED 0x1280 #define CL_PROFILING_COMMAND_SUBMIT 0x1281 #define CL_PROFILING_COMMAND_START 0x1282 #define CL_PROFILING_COMMAND_END 0x1283 +#ifdef CL_VERSION_2_0 #define CL_PROFILING_COMMAND_COMPLETE 0x1284 +#endif + +#ifdef CL_EXPERIMENTAL + +/* cl_device_atomic_capabilities - bitfield */ +#define CL_DEVICE_ATOMIC_ORDER_RELAXED (1 << 0) +#define CL_DEVICE_ATOMIC_ORDER_ACQ_REL (1 << 1) +#define CL_DEVICE_ATOMIC_ORDER_SEQ_CST (1 << 2) +#define CL_DEVICE_ATOMIC_SCOPE_WORK_ITEM (1 << 3) +#define CL_DEVICE_ATOMIC_SCOPE_WORK_GROUP (1 << 4) +#define CL_DEVICE_ATOMIC_SCOPE_DEVICE (1 << 5) +#define CL_DEVICE_ATOMIC_SCOPE_ALL_SVM_DEVICES (1 << 6) + +/* cl_device_info */ +#define CL_DEVICE_ATOMIC_MEMORY_CAPABILITIES 0x1063 +#define CL_DEVICE_ATOMIC_FENCE_CAPABILITIES 0x1064 +#define CL_DEVICE_NON_UNIFORM_WORK_GROUP_SUPPORT 0x1065 +#define CL_DEVICE_OPENCL_C_VERSIONS 0x1066 +#define CL_DEVICE_MAX_WRITE_IMAGE3D_ARGS 0x1067 +#define CL_DEVICE_WORK_GROUP_COLLECTIVE_FUNCTIONS_SUPPORT 0x1068 +#define CL_DEVICE_GENERIC_ADDRESS_SPACE_SUPPORT 0x1069 +/* 0x106A to 0x106E - Reserved for upcoming KHR extension */ +#define CL_DEVICE_OPENCL_C_FEATURES 0x106F + +/* cl_command_type */ +#define CL_COMMAND_SVM_MIGRATE_MEM 0x120E + +#endif + +/* cl_khronos_vendor_id */ +#define CL_KHRONOS_VENDOR_ID_CODEPLAY 0x10004 /********************************************************************************************************/ /* Platform API */ extern CL_API_ENTRY cl_int CL_API_CALL -clGetPlatformIDs(cl_uint /* num_entries */, - cl_platform_id * /* platforms */, - cl_uint * /* num_platforms */) CL_API_SUFFIX__VERSION_1_0; +clGetPlatformIDs(cl_uint num_entries, + cl_platform_id * platforms, + cl_uint * num_platforms) CL_API_SUFFIX__VERSION_1_0; -extern CL_API_ENTRY cl_int CL_API_CALL -clGetPlatformInfo(cl_platform_id /* platform */, - cl_platform_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; +extern CL_API_ENTRY cl_int CL_API_CALL +clGetPlatformInfo(cl_platform_id platform, + cl_platform_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; /* Device APIs */ extern CL_API_ENTRY cl_int CL_API_CALL -clGetDeviceIDs(cl_platform_id /* platform */, - cl_device_type /* device_type */, - cl_uint /* num_entries */, - cl_device_id * /* devices */, - cl_uint * /* num_devices */) CL_API_SUFFIX__VERSION_1_0; +clGetDeviceIDs(cl_platform_id platform, + cl_device_type device_type, + cl_uint num_entries, + cl_device_id * devices, + cl_uint * num_devices) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetDeviceInfo(cl_device_id /* device */, - cl_device_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clCreateSubDevices(cl_device_id /* in_device */, - const cl_device_partition_property * /* properties */, - cl_uint /* num_devices */, - cl_device_id * /* out_devices */, - cl_uint * /* num_devices_ret */) CL_API_SUFFIX__VERSION_1_2; +clGetDeviceInfo(cl_device_id device, + cl_device_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_int CL_API_CALL -clRetainDevice(cl_device_id /* device */) CL_API_SUFFIX__VERSION_1_2; - -extern CL_API_ENTRY cl_int CL_API_CALL -clReleaseDevice(cl_device_id /* device */) CL_API_SUFFIX__VERSION_1_2; +clCreateSubDevices(cl_device_id in_device, + const cl_device_partition_property * properties, + cl_uint num_devices, + cl_device_id * out_devices, + cl_uint * num_devices_ret) CL_API_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clSetDefaultDeviceCommandQueue(cl_context /* context */, - cl_device_id /* device */, - cl_command_queue /* command_queue */) CL_API_SUFFIX__VERSION_2_1; +clRetainDevice(cl_device_id device) CL_API_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clGetDeviceAndHostTimer(cl_device_id /* device */, - cl_ulong* /* device_timestamp */, - cl_ulong* /* host_timestamp */) CL_API_SUFFIX__VERSION_2_1; +clReleaseDevice(cl_device_id device) CL_API_SUFFIX__VERSION_1_2; + +#endif + +#ifdef CL_VERSION_2_1 extern CL_API_ENTRY cl_int CL_API_CALL -clGetHostTimer(cl_device_id /* device */, - cl_ulong * /* host_timestamp */) CL_API_SUFFIX__VERSION_2_1; +clSetDefaultDeviceCommandQueue(cl_context context, + cl_device_id device, + cl_command_queue command_queue) CL_API_SUFFIX__VERSION_2_1; - -/* Context APIs */ +extern CL_API_ENTRY cl_int CL_API_CALL +clGetDeviceAndHostTimer(cl_device_id device, + cl_ulong* device_timestamp, + cl_ulong* host_timestamp) CL_API_SUFFIX__VERSION_2_1; + +extern CL_API_ENTRY cl_int CL_API_CALL +clGetHostTimer(cl_device_id device, + cl_ulong * host_timestamp) CL_API_SUFFIX__VERSION_2_1; + +#endif + +/* Context APIs */ extern CL_API_ENTRY cl_context CL_API_CALL -clCreateContext(const cl_context_properties * /* properties */, - cl_uint /* num_devices */, - const cl_device_id * /* devices */, - void (CL_CALLBACK * /* pfn_notify */)(const char *, const void *, size_t, void *), - void * /* user_data */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateContext(const cl_context_properties * properties, + cl_uint num_devices, + const cl_device_id * devices, + void (CL_CALLBACK * pfn_notify)(const char * errinfo, + const void * private_info, + size_t cb, + void * user_data), + void * user_data, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_context CL_API_CALL -clCreateContextFromType(const cl_context_properties * /* properties */, - cl_device_type /* device_type */, - void (CL_CALLBACK * /* pfn_notify*/ )(const char *, const void *, size_t, void *), - void * /* user_data */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateContextFromType(const cl_context_properties * properties, + cl_device_type device_type, + void (CL_CALLBACK * pfn_notify)(const char * errinfo, + const void * private_info, + size_t cb, + void * user_data), + void * user_data, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clRetainContext(cl_context /* context */) CL_API_SUFFIX__VERSION_1_0; +clRetainContext(cl_context context) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clReleaseContext(cl_context /* context */) CL_API_SUFFIX__VERSION_1_0; +clReleaseContext(cl_context context) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetContextInfo(cl_context /* context */, - cl_context_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; +clGetContextInfo(cl_context context, + cl_context_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; /* Command Queue APIs */ + +#ifdef CL_VERSION_2_0 + extern CL_API_ENTRY cl_command_queue CL_API_CALL -clCreateCommandQueueWithProperties(cl_context /* context */, - cl_device_id /* device */, - const cl_queue_properties * /* properties */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; +clCreateCommandQueueWithProperties(cl_context context, + cl_device_id device, + const cl_queue_properties * properties, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_2_0; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clRetainCommandQueue(cl_command_queue /* command_queue */) CL_API_SUFFIX__VERSION_1_0; +clRetainCommandQueue(cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clReleaseCommandQueue(cl_command_queue /* command_queue */) CL_API_SUFFIX__VERSION_1_0; +clReleaseCommandQueue(cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetCommandQueueInfo(cl_command_queue /* command_queue */, - cl_command_queue_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; +clGetCommandQueueInfo(cl_command_queue command_queue, + cl_command_queue_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; /* Memory Object APIs */ extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateBuffer(cl_context /* context */, - cl_mem_flags /* flags */, - size_t /* size */, - void * /* host_ptr */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateBuffer(cl_context context, + cl_mem_flags flags, + size_t size, + void * host_ptr, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateSubBuffer(cl_mem /* buffer */, - cl_mem_flags /* flags */, - cl_buffer_create_type /* buffer_create_type */, - const void * /* buffer_create_info */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_1; +clCreateSubBuffer(cl_mem buffer, + cl_mem_flags flags, + cl_buffer_create_type buffer_create_type, + const void * buffer_create_info, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_1; + +#endif + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateImage(cl_context /* context */, - cl_mem_flags /* flags */, - const cl_image_format * /* image_format */, - const cl_image_desc * /* image_desc */, - void * /* host_ptr */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_2; - +clCreateImage(cl_context context, + cl_mem_flags flags, + const cl_image_format * image_format, + const cl_image_desc * image_desc, + void * host_ptr, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +#endif + +#ifdef CL_VERSION_2_0 + extern CL_API_ENTRY cl_mem CL_API_CALL -clCreatePipe(cl_context /* context */, - cl_mem_flags /* flags */, - cl_uint /* pipe_packet_size */, - cl_uint /* pipe_max_packets */, - const cl_pipe_properties * /* properties */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clRetainMemObject(cl_mem /* memobj */) CL_API_SUFFIX__VERSION_1_0; +clCreatePipe(cl_context context, + cl_mem_flags flags, + cl_uint pipe_packet_size, + cl_uint pipe_max_packets, + const cl_pipe_properties * properties, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_2_0; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clReleaseMemObject(cl_mem /* memobj */) CL_API_SUFFIX__VERSION_1_0; +clRetainMemObject(cl_mem memobj) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetSupportedImageFormats(cl_context /* context */, - cl_mem_flags /* flags */, - cl_mem_object_type /* image_type */, - cl_uint /* num_entries */, - cl_image_format * /* image_formats */, - cl_uint * /* num_image_formats */) CL_API_SUFFIX__VERSION_1_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clGetMemObjectInfo(cl_mem /* memobj */, - cl_mem_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; +clReleaseMemObject(cl_mem memobj) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetImageInfo(cl_mem /* image */, - cl_image_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clGetPipeInfo(cl_mem /* pipe */, - cl_pipe_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_2_0; - +clGetSupportedImageFormats(cl_context context, + cl_mem_flags flags, + cl_mem_object_type image_type, + cl_uint num_entries, + cl_image_format * image_formats, + cl_uint * num_image_formats) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clSetMemObjectDestructorCallback(cl_mem /* memobj */, - void (CL_CALLBACK * /*pfn_notify*/)( cl_mem /* memobj */, void* /*user_data*/), - void * /*user_data */ ) CL_API_SUFFIX__VERSION_1_1; +clGetMemObjectInfo(cl_mem memobj, + cl_mem_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +extern CL_API_ENTRY cl_int CL_API_CALL +clGetImageInfo(cl_mem image, + cl_image_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_2_0 + +extern CL_API_ENTRY cl_int CL_API_CALL +clGetPipeInfo(cl_mem pipe, + cl_pipe_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_2_0; + +#endif + +#ifdef CL_VERSION_1_1 + +extern CL_API_ENTRY cl_int CL_API_CALL +clSetMemObjectDestructorCallback(cl_mem memobj, + void (CL_CALLBACK * pfn_notify)(cl_mem memobj, + void * user_data), + void * user_data) CL_API_SUFFIX__VERSION_1_1; + +#endif /* SVM Allocation APIs */ + +#ifdef CL_VERSION_2_0 + extern CL_API_ENTRY void * CL_API_CALL -clSVMAlloc(cl_context /* context */, - cl_svm_mem_flags /* flags */, - size_t /* size */, - cl_uint /* alignment */) CL_API_SUFFIX__VERSION_2_0; +clSVMAlloc(cl_context context, + cl_svm_mem_flags flags, + size_t size, + cl_uint alignment) CL_API_SUFFIX__VERSION_2_0; extern CL_API_ENTRY void CL_API_CALL -clSVMFree(cl_context /* context */, - void * /* svm_pointer */) CL_API_SUFFIX__VERSION_2_0; - +clSVMFree(cl_context context, + void * svm_pointer) CL_API_SUFFIX__VERSION_2_0; + +#endif + /* Sampler APIs */ + +#ifdef CL_VERSION_2_0 + extern CL_API_ENTRY cl_sampler CL_API_CALL -clCreateSamplerWithProperties(cl_context /* context */, - const cl_sampler_properties * /* normalized_coords */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; +clCreateSamplerWithProperties(cl_context context, + const cl_sampler_properties * sampler_properties, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_2_0; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clRetainSampler(cl_sampler /* sampler */) CL_API_SUFFIX__VERSION_1_0; +clRetainSampler(cl_sampler sampler) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clReleaseSampler(cl_sampler /* sampler */) CL_API_SUFFIX__VERSION_1_0; +clReleaseSampler(cl_sampler sampler) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetSamplerInfo(cl_sampler /* sampler */, - cl_sampler_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; - -/* Program Object APIs */ +clGetSamplerInfo(cl_sampler sampler, + cl_sampler_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +/* Program Object APIs */ extern CL_API_ENTRY cl_program CL_API_CALL -clCreateProgramWithSource(cl_context /* context */, - cl_uint /* count */, - const char ** /* strings */, - const size_t * /* lengths */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateProgramWithSource(cl_context context, + cl_uint count, + const char ** strings, + const size_t * lengths, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_program CL_API_CALL -clCreateProgramWithBinary(cl_context /* context */, - cl_uint /* num_devices */, - const cl_device_id * /* device_list */, - const size_t * /* lengths */, - const unsigned char ** /* binaries */, - cl_int * /* binary_status */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateProgramWithBinary(cl_context context, + cl_uint num_devices, + const cl_device_id * device_list, + const size_t * lengths, + const unsigned char ** binaries, + cl_int * binary_status, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_program CL_API_CALL -clCreateProgramWithBuiltInKernels(cl_context /* context */, - cl_uint /* num_devices */, - const cl_device_id * /* device_list */, - const char * /* kernel_names */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_2; +clCreateProgramWithBuiltInKernels(cl_context context, + cl_uint num_devices, + const cl_device_id * device_list, + const char * kernel_names, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +#endif + +#ifdef CL_VERSION_2_1 extern CL_API_ENTRY cl_program CL_API_CALL -clCreateProgramWithIL(cl_context /* context */, - const void* /* il */, - size_t /* length */, - cl_int* /* errcode_ret */) CL_API_SUFFIX__VERSION_2_1; +clCreateProgramWithIL(cl_context context, + const void* il, + size_t length, + cl_int* errcode_ret) CL_API_SUFFIX__VERSION_2_1; +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clRetainProgram(cl_program /* program */) CL_API_SUFFIX__VERSION_1_0; +clRetainProgram(cl_program program) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clReleaseProgram(cl_program /* program */) CL_API_SUFFIX__VERSION_1_0; +clReleaseProgram(cl_program program) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clBuildProgram(cl_program /* program */, - cl_uint /* num_devices */, - const cl_device_id * /* device_list */, - const char * /* options */, - void (CL_CALLBACK * /* pfn_notify */)(cl_program /* program */, void * /* user_data */), - void * /* user_data */) CL_API_SUFFIX__VERSION_1_0; +clBuildProgram(cl_program program, + cl_uint num_devices, + const cl_device_id * device_list, + const char * options, + void (CL_CALLBACK * pfn_notify)(cl_program program, + void * user_data), + void * user_data) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_int CL_API_CALL -clCompileProgram(cl_program /* program */, - cl_uint /* num_devices */, - const cl_device_id * /* device_list */, - const char * /* options */, - cl_uint /* num_input_headers */, - const cl_program * /* input_headers */, - const char ** /* header_include_names */, - void (CL_CALLBACK * /* pfn_notify */)(cl_program /* program */, void * /* user_data */), - void * /* user_data */) CL_API_SUFFIX__VERSION_1_2; +clCompileProgram(cl_program program, + cl_uint num_devices, + const cl_device_id * device_list, + const char * options, + cl_uint num_input_headers, + const cl_program * input_headers, + const char ** header_include_names, + void (CL_CALLBACK * pfn_notify)(cl_program program, + void * user_data), + void * user_data) CL_API_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_program CL_API_CALL -clLinkProgram(cl_context /* context */, - cl_uint /* num_devices */, - const cl_device_id * /* device_list */, - const char * /* options */, - cl_uint /* num_input_programs */, - const cl_program * /* input_programs */, - void (CL_CALLBACK * /* pfn_notify */)(cl_program /* program */, void * /* user_data */), - void * /* user_data */, - cl_int * /* errcode_ret */ ) CL_API_SUFFIX__VERSION_1_2; +clLinkProgram(cl_context context, + cl_uint num_devices, + const cl_device_id * device_list, + const char * options, + cl_uint num_input_programs, + const cl_program * input_programs, + void (CL_CALLBACK * pfn_notify)(cl_program program, + void * user_data), + void * user_data, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +#endif + +#ifdef CL_VERSION_2_2 extern CL_API_ENTRY cl_int CL_API_CALL -clSetProgramReleaseCallback(cl_program /* program */, - void (CL_CALLBACK * /* pfn_notify */)(cl_program /* program */, void * /* user_data */), - void * /* user_data */) CL_API_SUFFIX__VERSION_2_2; +clSetProgramReleaseCallback(cl_program program, + void (CL_CALLBACK * pfn_notify)(cl_program program, + void * user_data), + void * user_data) CL_API_SUFFIX__VERSION_2_2; extern CL_API_ENTRY cl_int CL_API_CALL -clSetProgramSpecializationConstant(cl_program /* program */, - cl_uint /* spec_id */, - size_t /* spec_size */, - const void* /* spec_value */) CL_API_SUFFIX__VERSION_2_2; +clSetProgramSpecializationConstant(cl_program program, + cl_uint spec_id, + size_t spec_size, + const void* spec_value) CL_API_SUFFIX__VERSION_2_2; +#endif + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_int CL_API_CALL -clUnloadPlatformCompiler(cl_platform_id /* platform */) CL_API_SUFFIX__VERSION_1_2; +clUnloadPlatformCompiler(cl_platform_id platform) CL_API_SUFFIX__VERSION_1_2; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clGetProgramInfo(cl_program /* program */, - cl_program_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; +clGetProgramInfo(cl_program program, + cl_program_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetProgramBuildInfo(cl_program /* program */, - cl_device_id /* device */, - cl_program_build_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; - +clGetProgramBuildInfo(cl_program program, + cl_device_id device, + cl_program_build_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + /* Kernel Object APIs */ extern CL_API_ENTRY cl_kernel CL_API_CALL -clCreateKernel(cl_program /* program */, - const char * /* kernel_name */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateKernel(cl_program program, + const char * kernel_name, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clCreateKernelsInProgram(cl_program /* program */, - cl_uint /* num_kernels */, - cl_kernel * /* kernels */, - cl_uint * /* num_kernels_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateKernelsInProgram(cl_program program, + cl_uint num_kernels, + cl_kernel * kernels, + cl_uint * num_kernels_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_2_1 extern CL_API_ENTRY cl_kernel CL_API_CALL -clCloneKernel(cl_kernel /* source_kernel */, - cl_int* /* errcode_ret */) CL_API_SUFFIX__VERSION_2_1; +clCloneKernel(cl_kernel source_kernel, + cl_int* errcode_ret) CL_API_SUFFIX__VERSION_2_1; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clRetainKernel(cl_kernel /* kernel */) CL_API_SUFFIX__VERSION_1_0; +clRetainKernel(cl_kernel kernel) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clReleaseKernel(cl_kernel /* kernel */) CL_API_SUFFIX__VERSION_1_0; +clReleaseKernel(cl_kernel kernel) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clSetKernelArg(cl_kernel /* kernel */, - cl_uint /* arg_index */, - size_t /* arg_size */, - const void * /* arg_value */) CL_API_SUFFIX__VERSION_1_0; +clSetKernelArg(cl_kernel kernel, + cl_uint arg_index, + size_t arg_size, + const void * arg_value) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_2_0 extern CL_API_ENTRY cl_int CL_API_CALL -clSetKernelArgSVMPointer(cl_kernel /* kernel */, - cl_uint /* arg_index */, - const void * /* arg_value */) CL_API_SUFFIX__VERSION_2_0; +clSetKernelArgSVMPointer(cl_kernel kernel, + cl_uint arg_index, + const void * arg_value) CL_API_SUFFIX__VERSION_2_0; extern CL_API_ENTRY cl_int CL_API_CALL -clSetKernelExecInfo(cl_kernel /* kernel */, - cl_kernel_exec_info /* param_name */, - size_t /* param_value_size */, - const void * /* param_value */) CL_API_SUFFIX__VERSION_2_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clGetKernelInfo(cl_kernel /* kernel */, - cl_kernel_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; +clSetKernelExecInfo(cl_kernel kernel, + cl_kernel_exec_info param_name, + size_t param_value_size, + const void * param_value) CL_API_SUFFIX__VERSION_2_0; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clGetKernelArgInfo(cl_kernel /* kernel */, - cl_uint /* arg_indx */, - cl_kernel_arg_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_2; +clGetKernelInfo(cl_kernel kernel, + cl_kernel_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_int CL_API_CALL -clGetKernelWorkGroupInfo(cl_kernel /* kernel */, - cl_device_id /* device */, - cl_kernel_work_group_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; +clGetKernelArgInfo(cl_kernel kernel, + cl_uint arg_indx, + cl_kernel_arg_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_2; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clGetKernelSubGroupInfo(cl_kernel /* kernel */, - cl_device_id /* device */, - cl_kernel_sub_group_info /* param_name */, - size_t /* input_value_size */, - const void* /*input_value */, - size_t /* param_value_size */, - void* /* param_value */, - size_t* /* param_value_size_ret */ ) CL_API_SUFFIX__VERSION_2_1; +clGetKernelWorkGroupInfo(cl_kernel kernel, + cl_device_id device, + cl_kernel_work_group_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; +#ifdef CL_VERSION_2_1 + +extern CL_API_ENTRY cl_int CL_API_CALL +clGetKernelSubGroupInfo(cl_kernel kernel, + cl_device_id device, + cl_kernel_sub_group_info param_name, + size_t input_value_size, + const void* input_value, + size_t param_value_size, + void* param_value, + size_t* param_value_size_ret) CL_API_SUFFIX__VERSION_2_1; + +#endif /* Event Object APIs */ extern CL_API_ENTRY cl_int CL_API_CALL -clWaitForEvents(cl_uint /* num_events */, - const cl_event * /* event_list */) CL_API_SUFFIX__VERSION_1_0; +clWaitForEvents(cl_uint num_events, + const cl_event * event_list) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetEventInfo(cl_event /* event */, - cl_event_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; - +clGetEventInfo(cl_event event, + cl_event_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 + extern CL_API_ENTRY cl_event CL_API_CALL -clCreateUserEvent(cl_context /* context */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_1; - -extern CL_API_ENTRY cl_int CL_API_CALL -clRetainEvent(cl_event /* event */) CL_API_SUFFIX__VERSION_1_0; +clCreateUserEvent(cl_context context, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_1; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clReleaseEvent(cl_event /* event */) CL_API_SUFFIX__VERSION_1_0; +clRetainEvent(cl_event event) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clSetUserEventStatus(cl_event /* event */, - cl_int /* execution_status */) CL_API_SUFFIX__VERSION_1_1; - +clReleaseEvent(cl_event event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 + extern CL_API_ENTRY cl_int CL_API_CALL -clSetEventCallback( cl_event /* event */, - cl_int /* command_exec_callback_type */, - void (CL_CALLBACK * /* pfn_notify */)(cl_event, cl_int, void *), - void * /* user_data */) CL_API_SUFFIX__VERSION_1_1; +clSetUserEventStatus(cl_event event, + cl_int execution_status) CL_API_SUFFIX__VERSION_1_1; + +extern CL_API_ENTRY cl_int CL_API_CALL +clSetEventCallback(cl_event event, + cl_int command_exec_callback_type, + void (CL_CALLBACK * pfn_notify)(cl_event event, + cl_int event_command_status, + void * user_data), + void * user_data) CL_API_SUFFIX__VERSION_1_1; + +#endif /* Profiling APIs */ extern CL_API_ENTRY cl_int CL_API_CALL -clGetEventProfilingInfo(cl_event /* event */, - cl_profiling_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; - +clGetEventProfilingInfo(cl_event event, + cl_profiling_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + /* Flush and Finish APIs */ extern CL_API_ENTRY cl_int CL_API_CALL -clFlush(cl_command_queue /* command_queue */) CL_API_SUFFIX__VERSION_1_0; +clFlush(cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clFinish(cl_command_queue /* command_queue */) CL_API_SUFFIX__VERSION_1_0; +clFinish(cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; /* Enqueued Commands APIs */ extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReadBuffer(cl_command_queue /* command_queue */, - cl_mem /* buffer */, - cl_bool /* blocking_read */, - size_t /* offset */, - size_t /* size */, - void * /* ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReadBufferRect(cl_command_queue /* command_queue */, - cl_mem /* buffer */, - cl_bool /* blocking_read */, - const size_t * /* buffer_offset */, - const size_t * /* host_offset */, - const size_t * /* region */, - size_t /* buffer_row_pitch */, - size_t /* buffer_slice_pitch */, - size_t /* host_row_pitch */, - size_t /* host_slice_pitch */, - void * /* ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_1; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueWriteBuffer(cl_command_queue /* command_queue */, - cl_mem /* buffer */, - cl_bool /* blocking_write */, - size_t /* offset */, - size_t /* size */, - const void * /* ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueWriteBufferRect(cl_command_queue /* command_queue */, - cl_mem /* buffer */, - cl_bool /* blocking_write */, - const size_t * /* buffer_offset */, - const size_t * /* host_offset */, - const size_t * /* region */, - size_t /* buffer_row_pitch */, - size_t /* buffer_slice_pitch */, - size_t /* host_row_pitch */, - size_t /* host_slice_pitch */, - const void * /* ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_1; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueFillBuffer(cl_command_queue /* command_queue */, - cl_mem /* buffer */, - const void * /* pattern */, - size_t /* pattern_size */, - size_t /* offset */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_2; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueCopyBuffer(cl_command_queue /* command_queue */, - cl_mem /* src_buffer */, - cl_mem /* dst_buffer */, - size_t /* src_offset */, - size_t /* dst_offset */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueCopyBufferRect(cl_command_queue /* command_queue */, - cl_mem /* src_buffer */, - cl_mem /* dst_buffer */, - const size_t * /* src_origin */, - const size_t * /* dst_origin */, - const size_t * /* region */, - size_t /* src_row_pitch */, - size_t /* src_slice_pitch */, - size_t /* dst_row_pitch */, - size_t /* dst_slice_pitch */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_1; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReadImage(cl_command_queue /* command_queue */, - cl_mem /* image */, - cl_bool /* blocking_read */, - const size_t * /* origin[3] */, - const size_t * /* region[3] */, - size_t /* row_pitch */, - size_t /* slice_pitch */, - void * /* ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueReadBuffer(cl_command_queue command_queue, + cl_mem buffer, + cl_bool blocking_read, + size_t offset, + size_t size, + void * ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueWriteImage(cl_command_queue /* command_queue */, - cl_mem /* image */, - cl_bool /* blocking_write */, - const size_t * /* origin[3] */, - const size_t * /* region[3] */, - size_t /* input_row_pitch */, - size_t /* input_slice_pitch */, - const void * /* ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueReadBufferRect(cl_command_queue command_queue, + cl_mem buffer, + cl_bool blocking_read, + const size_t * buffer_offset, + const size_t * host_offset, + const size_t * region, + size_t buffer_row_pitch, + size_t buffer_slice_pitch, + size_t host_row_pitch, + size_t host_slice_pitch, + void * ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_1; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueFillImage(cl_command_queue /* command_queue */, - cl_mem /* image */, - const void * /* fill_color */, - const size_t * /* origin[3] */, - const size_t * /* region[3] */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_2; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueCopyImage(cl_command_queue /* command_queue */, - cl_mem /* src_image */, - cl_mem /* dst_image */, - const size_t * /* src_origin[3] */, - const size_t * /* dst_origin[3] */, - const size_t * /* region[3] */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueWriteBuffer(cl_command_queue command_queue, + cl_mem buffer, + cl_bool blocking_write, + size_t offset, + size_t size, + const void * ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueCopyImageToBuffer(cl_command_queue /* command_queue */, - cl_mem /* src_image */, - cl_mem /* dst_buffer */, - const size_t * /* src_origin[3] */, - const size_t * /* region[3] */, - size_t /* dst_offset */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueWriteBufferRect(cl_command_queue command_queue, + cl_mem buffer, + cl_bool blocking_write, + const size_t * buffer_offset, + const size_t * host_offset, + const size_t * region, + size_t buffer_row_pitch, + size_t buffer_slice_pitch, + size_t host_row_pitch, + size_t host_slice_pitch, + const void * ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_1; + +#endif + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueCopyBufferToImage(cl_command_queue /* command_queue */, - cl_mem /* src_buffer */, - cl_mem /* dst_image */, - size_t /* src_offset */, - const size_t * /* dst_origin[3] */, - const size_t * /* region[3] */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueFillBuffer(cl_command_queue command_queue, + cl_mem buffer, + const void * pattern, + size_t pattern_size, + size_t offset, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_2; + +#endif + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueCopyBuffer(cl_command_queue command_queue, + cl_mem src_buffer, + cl_mem dst_buffer, + size_t src_offset, + size_t dst_offset, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueCopyBufferRect(cl_command_queue command_queue, + cl_mem src_buffer, + cl_mem dst_buffer, + const size_t * src_origin, + const size_t * dst_origin, + const size_t * region, + size_t src_row_pitch, + size_t src_slice_pitch, + size_t dst_row_pitch, + size_t dst_slice_pitch, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_1; + +#endif + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueReadImage(cl_command_queue command_queue, + cl_mem image, + cl_bool blocking_read, + const size_t * origin, + const size_t * region, + size_t row_pitch, + size_t slice_pitch, + void * ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueWriteImage(cl_command_queue command_queue, + cl_mem image, + cl_bool blocking_write, + const size_t * origin, + const size_t * region, + size_t input_row_pitch, + size_t input_slice_pitch, + const void * ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueFillImage(cl_command_queue command_queue, + cl_mem image, + const void * fill_color, + const size_t * origin, + const size_t * region, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_2; + +#endif + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueCopyImage(cl_command_queue command_queue, + cl_mem src_image, + cl_mem dst_image, + const size_t * src_origin, + const size_t * dst_origin, + const size_t * region, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueCopyImageToBuffer(cl_command_queue command_queue, + cl_mem src_image, + cl_mem dst_buffer, + const size_t * src_origin, + const size_t * region, + size_t dst_offset, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueCopyBufferToImage(cl_command_queue command_queue, + cl_mem src_buffer, + cl_mem dst_image, + size_t src_offset, + const size_t * dst_origin, + const size_t * region, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY void * CL_API_CALL -clEnqueueMapBuffer(cl_command_queue /* command_queue */, - cl_mem /* buffer */, - cl_bool /* blocking_map */, - cl_map_flags /* map_flags */, - size_t /* offset */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueMapBuffer(cl_command_queue command_queue, + cl_mem buffer, + cl_bool blocking_map, + cl_map_flags map_flags, + size_t offset, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY void * CL_API_CALL -clEnqueueMapImage(cl_command_queue /* command_queue */, - cl_mem /* image */, - cl_bool /* blocking_map */, - cl_map_flags /* map_flags */, - const size_t * /* origin[3] */, - const size_t * /* region[3] */, - size_t * /* image_row_pitch */, - size_t * /* image_slice_pitch */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueMapImage(cl_command_queue command_queue, + cl_mem image, + cl_bool blocking_map, + cl_map_flags map_flags, + const size_t * origin, + const size_t * region, + size_t * image_row_pitch, + size_t * image_slice_pitch, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueUnmapMemObject(cl_command_queue /* command_queue */, - cl_mem /* memobj */, - void * /* mapped_ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueUnmapMemObject(cl_command_queue command_queue, + cl_mem memobj, + void * mapped_ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueMigrateMemObjects(cl_command_queue /* command_queue */, - cl_uint /* num_mem_objects */, - const cl_mem * /* mem_objects */, - cl_mem_migration_flags /* flags */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_2; +clEnqueueMigrateMemObjects(cl_command_queue command_queue, + cl_uint num_mem_objects, + const cl_mem * mem_objects, + cl_mem_migration_flags flags, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_2; + +#endif extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueNDRangeKernel(cl_command_queue /* command_queue */, - cl_kernel /* kernel */, - cl_uint /* work_dim */, - const size_t * /* global_work_offset */, - const size_t * /* global_work_size */, - const size_t * /* local_work_size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueNDRangeKernel(cl_command_queue command_queue, + cl_kernel kernel, + cl_uint work_dim, + const size_t * global_work_offset, + const size_t * global_work_size, + const size_t * local_work_size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueNativeKernel(cl_command_queue /* command_queue */, - void (CL_CALLBACK * /*user_func*/)(void *), - void * /* args */, - size_t /* cb_args */, - cl_uint /* num_mem_objects */, - const cl_mem * /* mem_list */, - const void ** /* args_mem_loc */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueNativeKernel(cl_command_queue command_queue, + void (CL_CALLBACK * user_func)(void *), + void * args, + size_t cb_args, + cl_uint num_mem_objects, + const cl_mem * mem_list, + const void ** args_mem_loc, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueMarkerWithWaitList(cl_command_queue /* command_queue */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_2; +clEnqueueMarkerWithWaitList(cl_command_queue command_queue, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueBarrierWithWaitList(cl_command_queue /* command_queue */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_2; +clEnqueueBarrierWithWaitList(cl_command_queue command_queue, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_2; + +#endif + +#ifdef CL_VERSION_2_0 extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMFree(cl_command_queue /* command_queue */, - cl_uint /* num_svm_pointers */, - void *[] /* svm_pointers[] */, - void (CL_CALLBACK * /*pfn_free_func*/)(cl_command_queue /* queue */, - cl_uint /* num_svm_pointers */, - void *[] /* svm_pointers[] */, - void * /* user_data */), - void * /* user_data */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; +clEnqueueSVMFree(cl_command_queue command_queue, + cl_uint num_svm_pointers, + void * svm_pointers[], + void (CL_CALLBACK * pfn_free_func)(cl_command_queue queue, + cl_uint num_svm_pointers, + void * svm_pointers[], + void * user_data), + void * user_data, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_2_0; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMMemcpy(cl_command_queue /* command_queue */, - cl_bool /* blocking_copy */, - void * /* dst_ptr */, - const void * /* src_ptr */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; +clEnqueueSVMMemcpy(cl_command_queue command_queue, + cl_bool blocking_copy, + void * dst_ptr, + const void * src_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_2_0; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMMemFill(cl_command_queue /* command_queue */, - void * /* svm_ptr */, - const void * /* pattern */, - size_t /* pattern_size */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMMap(cl_command_queue /* command_queue */, - cl_bool /* blocking_map */, - cl_map_flags /* flags */, - void * /* svm_ptr */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMUnmap(cl_command_queue /* command_queue */, - void * /* svm_ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; +clEnqueueSVMMemFill(cl_command_queue command_queue, + void * svm_ptr, + const void * pattern, + size_t pattern_size, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_2_0; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMMigrateMem(cl_command_queue /* command_queue */, - cl_uint /* num_svm_pointers */, - const void ** /* svm_pointers */, - const size_t * /* sizes */, - cl_mem_migration_flags /* flags */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_1; +clEnqueueSVMMap(cl_command_queue command_queue, + cl_bool blocking_map, + cl_map_flags flags, + void * svm_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_2_0; +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueSVMUnmap(cl_command_queue command_queue, + void * svm_ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_2_0; + +#endif + +#ifdef CL_VERSION_2_1 + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueSVMMigrateMem(cl_command_queue command_queue, + cl_uint num_svm_pointers, + const void ** svm_pointers, + const size_t * sizes, + cl_mem_migration_flags flags, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_2_1; + +#endif + +#ifdef CL_VERSION_1_2 /* Extension function access * * Returns the extension function address for the given function name, * or NULL if a valid function can not be found. The client must - * check to make sure the address is not NULL, before using or + * check to make sure the address is not NULL, before using or * calling the returned function address. */ -extern CL_API_ENTRY void * CL_API_CALL -clGetExtensionFunctionAddressForPlatform(cl_platform_id /* platform */, - const char * /* func_name */) CL_API_SUFFIX__VERSION_1_2; - +extern CL_API_ENTRY void * CL_API_CALL +clGetExtensionFunctionAddressForPlatform(cl_platform_id platform, + const char * func_name) CL_API_SUFFIX__VERSION_1_2; + +#endif + +#ifdef CL_USE_DEPRECATED_OPENCL_1_0_APIS + /* + * WARNING: + * This API introduces mutable state into the OpenCL implementation. It has been REMOVED + * to better facilitate thread safety. The 1.0 API is not thread safe. It is not tested by the + * OpenCL 1.1 conformance test, and consequently may not work or may not work dependably. + * It is likely to be non-performant. Use of this API is not advised. Use at your own risk. + * + * Software developers previously relying on this API are instructed to set the command queue + * properties when creating the queue, instead. + */ + extern CL_API_ENTRY cl_int CL_API_CALL + clSetCommandQueueProperty(cl_command_queue command_queue, + cl_command_queue_properties properties, + cl_bool enable, + cl_command_queue_properties * old_properties) CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED; +#endif /* CL_USE_DEPRECATED_OPENCL_1_0_APIS */ /* Deprecated OpenCL 1.1 APIs */ extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED cl_mem CL_API_CALL -clCreateImage2D(cl_context /* context */, - cl_mem_flags /* flags */, - const cl_image_format * /* image_format */, - size_t /* image_width */, - size_t /* image_height */, - size_t /* image_row_pitch */, - void * /* host_ptr */, - cl_int * /* errcode_ret */) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - +clCreateImage2D(cl_context context, + cl_mem_flags flags, + const cl_image_format * image_format, + size_t image_width, + size_t image_height, + size_t image_row_pitch, + void * host_ptr, + cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED cl_mem CL_API_CALL -clCreateImage3D(cl_context /* context */, - cl_mem_flags /* flags */, - const cl_image_format * /* image_format */, - size_t /* image_width */, - size_t /* image_height */, - size_t /* image_depth */, - size_t /* image_row_pitch */, - size_t /* image_slice_pitch */, - void * /* host_ptr */, - cl_int * /* errcode_ret */) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - +clCreateImage3D(cl_context context, + cl_mem_flags flags, + const cl_image_format * image_format, + size_t image_width, + size_t image_height, + size_t image_depth, + size_t image_row_pitch, + size_t image_slice_pitch, + void * host_ptr, + cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED cl_int CL_API_CALL -clEnqueueMarker(cl_command_queue /* command_queue */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - +clEnqueueMarker(cl_command_queue command_queue, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED cl_int CL_API_CALL -clEnqueueWaitForEvents(cl_command_queue /* command_queue */, - cl_uint /* num_events */, - const cl_event * /* event_list */) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - +clEnqueueWaitForEvents(cl_command_queue command_queue, + cl_uint num_events, + const cl_event * event_list) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED cl_int CL_API_CALL -clEnqueueBarrier(cl_command_queue /* command_queue */) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; +clEnqueueBarrier(cl_command_queue command_queue) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED cl_int CL_API_CALL clUnloadCompiler(void) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - + extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED void * CL_API_CALL -clGetExtensionFunctionAddress(const char * /* func_name */) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - +clGetExtensionFunctionAddress(const char * func_name) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + /* Deprecated OpenCL 2.0 APIs */ extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_2_DEPRECATED cl_command_queue CL_API_CALL -clCreateCommandQueue(cl_context /* context */, - cl_device_id /* device */, - cl_command_queue_properties /* properties */, - cl_int * /* errcode_ret */) CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED; - - +clCreateCommandQueue(cl_context context, + cl_device_id device, + cl_command_queue_properties properties, + cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED; + extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_2_DEPRECATED cl_sampler CL_API_CALL -clCreateSampler(cl_context /* context */, - cl_bool /* normalized_coords */, - cl_addressing_mode /* addressing_mode */, - cl_filter_mode /* filter_mode */, - cl_int * /* errcode_ret */) CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED; - +clCreateSampler(cl_context context, + cl_bool normalized_coords, + cl_addressing_mode addressing_mode, + cl_filter_mode filter_mode, + cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED; + extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_2_DEPRECATED cl_int CL_API_CALL -clEnqueueTask(cl_command_queue /* command_queue */, - cl_kernel /* kernel */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED; - +clEnqueueTask(cl_command_queue command_queue, + cl_kernel kernel, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED; + #ifdef __cplusplus } #endif #endif /* __OPENCL_CL_H */ - diff --git a/amdocl/CL/cl_egl.h b/amdocl/CL/cl_egl.h index a765bd5266..bc4d998eb3 100644 --- a/amdocl/CL/cl_egl.h +++ b/amdocl/CL/cl_egl.h @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2008-2015 The Khronos Group Inc. + * Copyright (c) 2008-2019 The Khronos Group Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and/or associated documentation files (the @@ -29,11 +29,7 @@ #ifndef __OPENCL_CL_EGL_H #define __OPENCL_CL_EGL_H -#ifdef __APPLE__ - -#else #include -#endif #ifdef __cplusplus extern "C" { @@ -65,69 +61,69 @@ typedef intptr_t cl_egl_image_properties_khr; #define cl_khr_egl_image 1 extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromEGLImageKHR(cl_context /* context */, - CLeglDisplayKHR /* egldisplay */, - CLeglImageKHR /* eglimage */, - cl_mem_flags /* flags */, - const cl_egl_image_properties_khr * /* properties */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateFromEGLImageKHR(cl_context context, + CLeglDisplayKHR egldisplay, + CLeglImageKHR eglimage, + cl_mem_flags flags, + const cl_egl_image_properties_khr * properties, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; typedef CL_API_ENTRY cl_mem (CL_API_CALL *clCreateFromEGLImageKHR_fn)( - cl_context context, - CLeglDisplayKHR egldisplay, - CLeglImageKHR eglimage, - cl_mem_flags flags, - const cl_egl_image_properties_khr * properties, - cl_int * errcode_ret); + cl_context context, + CLeglDisplayKHR egldisplay, + CLeglImageKHR eglimage, + cl_mem_flags flags, + const cl_egl_image_properties_khr * properties, + cl_int * errcode_ret); extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueAcquireEGLObjectsKHR(cl_command_queue /* command_queue */, - cl_uint /* num_objects */, - const cl_mem * /* mem_objects */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueAcquireEGLObjectsKHR(cl_command_queue command_queue, + cl_uint num_objects, + const cl_mem * mem_objects, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; typedef CL_API_ENTRY cl_int (CL_API_CALL *clEnqueueAcquireEGLObjectsKHR_fn)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event); + cl_command_queue command_queue, + cl_uint num_objects, + const cl_mem * mem_objects, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event); extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReleaseEGLObjectsKHR(cl_command_queue /* command_queue */, - cl_uint /* num_objects */, - const cl_mem * /* mem_objects */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueReleaseEGLObjectsKHR(cl_command_queue command_queue, + cl_uint num_objects, + const cl_mem * mem_objects, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; typedef CL_API_ENTRY cl_int (CL_API_CALL *clEnqueueReleaseEGLObjectsKHR_fn)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event); + cl_command_queue command_queue, + cl_uint num_objects, + const cl_mem * mem_objects, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event); #define cl_khr_egl_event 1 extern CL_API_ENTRY cl_event CL_API_CALL -clCreateEventFromEGLSyncKHR(cl_context /* context */, - CLeglSyncKHR /* sync */, - CLeglDisplayKHR /* display */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateEventFromEGLSyncKHR(cl_context context, + CLeglSyncKHR sync, + CLeglDisplayKHR display, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; typedef CL_API_ENTRY cl_event (CL_API_CALL *clCreateEventFromEGLSyncKHR_fn)( - cl_context context, - CLeglSyncKHR sync, - CLeglDisplayKHR display, - cl_int * errcode_ret); + cl_context context, + CLeglSyncKHR sync, + CLeglDisplayKHR display, + cl_int * errcode_ret); #ifdef __cplusplus } diff --git a/amdocl/CL/cl_ext.h b/amdocl/CL/cl_ext.h index eef6a1a8b0..4d6d8c093a 100644 --- a/amdocl/CL/cl_ext.h +++ b/amdocl/CL/cl_ext.h @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2008-2015 The Khronos Group Inc. + * Copyright (c) 2008-2019 The Khronos Group Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and/or associated documentation files (the @@ -26,8 +26,6 @@ * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. ******************************************************************************/ -/* $Revision: 11928 $ on $Date: 2010-07-13 09:04:56 -0700 (Tue, 13 Jul 2010) $ */ - /* cl_ext.h contains OpenCL extensions which don't have external */ /* (OpenGL, D3D) dependencies. */ @@ -38,11 +36,13 @@ extern "C" { #endif -#ifdef __APPLE__ - #include - #include -#else - #include +#include + +/* cl_khr_fp64 extension - no extension #define since it has no functions */ +/* CL_DEVICE_DOUBLE_FP_CONFIG is defined in CL.h for OpenCL >= 120 */ + +#if CL_TARGET_OPENCL_VERSION <= 110 +#define CL_DEVICE_DOUBLE_FP_CONFIG 0x1032 #endif /* cl_khr_fp16 extension - no extension #define since it has no functions */ @@ -52,12 +52,12 @@ extern "C" { * * Apple extension for use to manage externally allocated buffers used with cl_mem objects with CL_MEM_USE_HOST_PTR * - * Registers a user callback function that will be called when the memory object is deleted and its resources - * freed. Each call to clSetMemObjectCallbackFn registers the specified user callback function on a callback - * stack associated with memobj. The registered user callback functions are called in the reverse order in - * which they were registered. The user callback functions are called and then the memory object is deleted - * and its resources freed. This provides a mechanism for the application (and libraries) using memobj to be - * notified when the memory referenced by host_ptr, specified when the memory object is created and used as + * Registers a user callback function that will be called when the memory object is deleted and its resources + * freed. Each call to clSetMemObjectCallbackFn registers the specified user callback function on a callback + * stack associated with memobj. The registered user callback functions are called in the reverse order in + * which they were registered. The user callback functions are called and then the memory object is deleted + * and its resources freed. This provides a mechanism for the application (and libraries) using memobj to be + * notified when the memory referenced by host_ptr, specified when the memory object is created and used as * the storage bits for the memory object, can be reused or freed. * * The application may not call CL api's with the cl_mem object passed to the pfn_notify. @@ -66,9 +66,9 @@ extern "C" { * before using. */ #define cl_APPLE_SetMemObjectDestructor 1 -cl_int CL_API_ENTRY clSetMemObjectDestructorAPPLE( cl_mem /* memobj */, - void (* /*pfn_notify*/)( cl_mem /* memobj */, void* /*user_data*/), - void * /*user_data */ ) CL_EXT_SUFFIX__VERSION_1_0; +cl_int CL_API_ENTRY clSetMemObjectDestructorAPPLE( cl_mem memobj, + void (* pfn_notify)(cl_mem memobj, void * user_data), + void * user_data) CL_EXT_SUFFIX__VERSION_1_0; /* Context Logging Functions @@ -77,29 +77,29 @@ cl_int CL_API_ENTRY clSetMemObjectDestructorAPPLE( cl_mem /* memobj */, * Please check for the "cl_APPLE_ContextLoggingFunctions" extension using clGetDeviceInfo(CL_DEVICE_EXTENSIONS) * before using. * - * clLogMessagesToSystemLog fowards on all log messages to the Apple System Logger + * clLogMessagesToSystemLog forwards on all log messages to the Apple System Logger */ #define cl_APPLE_ContextLoggingFunctions 1 -extern void CL_API_ENTRY clLogMessagesToSystemLogAPPLE( const char * /* errstr */, - const void * /* private_info */, - size_t /* cb */, - void * /* user_data */ ) CL_EXT_SUFFIX__VERSION_1_0; +extern void CL_API_ENTRY clLogMessagesToSystemLogAPPLE( const char * errstr, + const void * private_info, + size_t cb, + void * user_data) CL_EXT_SUFFIX__VERSION_1_0; /* clLogMessagesToStdout sends all log messages to the file descriptor stdout */ -extern void CL_API_ENTRY clLogMessagesToStdoutAPPLE( const char * /* errstr */, - const void * /* private_info */, - size_t /* cb */, - void * /* user_data */ ) CL_EXT_SUFFIX__VERSION_1_0; +extern void CL_API_ENTRY clLogMessagesToStdoutAPPLE( const char * errstr, + const void * private_info, + size_t cb, + void * user_data) CL_EXT_SUFFIX__VERSION_1_0; /* clLogMessagesToStderr sends all log messages to the file descriptor stderr */ -extern void CL_API_ENTRY clLogMessagesToStderrAPPLE( const char * /* errstr */, - const void * /* private_info */, - size_t /* cb */, - void * /* user_data */ ) CL_EXT_SUFFIX__VERSION_1_0; +extern void CL_API_ENTRY clLogMessagesToStderrAPPLE( const char * errstr, + const void * private_info, + size_t cb, + void * user_data) CL_EXT_SUFFIX__VERSION_1_0; -/************************ -* cl_khr_icd extension * +/************************ +* cl_khr_icd extension * ************************/ #define cl_khr_icd 1 @@ -110,55 +110,92 @@ extern void CL_API_ENTRY clLogMessagesToStderrAPPLE( const char * /* errstr */ #define CL_PLATFORM_NOT_FOUND_KHR -1001 extern CL_API_ENTRY cl_int CL_API_CALL -clIcdGetPlatformIDsKHR(cl_uint /* num_entries */, - cl_platform_id * /* platforms */, - cl_uint * /* num_platforms */); +clIcdGetPlatformIDsKHR(cl_uint num_entries, + cl_platform_id * platforms, + cl_uint * num_platforms); -typedef CL_API_ENTRY cl_int (CL_API_CALL *clIcdGetPlatformIDsKHR_fn)( - cl_uint /* num_entries */, - cl_platform_id * /* platforms */, - cl_uint * /* num_platforms */); +typedef CL_API_ENTRY cl_int +(CL_API_CALL *clIcdGetPlatformIDsKHR_fn)(cl_uint num_entries, + cl_platform_id * platforms, + cl_uint * num_platforms); -/* Extension: cl_khr_image2D_buffer - * - * This extension allows a 2D image to be created from a cl_mem buffer without a copy. - * The type associated with a 2D image created from a buffer in an OpenCL program is image2d_t. - * Both the sampler and sampler-less read_image built-in functions are supported for 2D images - * and 2D images created from a buffer. Similarly, the write_image built-ins are also supported - * for 2D images created from a buffer. - * - * When the 2D image from buffer is created, the client must specify the width, - * height, image format (i.e. channel order and channel data type) and optionally the row pitch - * - * The pitch specified must be a multiple of CL_DEVICE_IMAGE_PITCH_ALIGNMENT pixels. - * The base address of the buffer must be aligned to CL_DEVICE_IMAGE_BASE_ADDRESS_ALIGNMENT pixels. +/******************************* + * cl_khr_il_program extension * + *******************************/ +#define cl_khr_il_program 1 + +/* New property to clGetDeviceInfo for retrieving supported intermediate + * languages */ - -/************************************* - * cl_khr_initalize_memory extension * - *************************************/ - +#define CL_DEVICE_IL_VERSION_KHR 0x105B + +/* New property to clGetProgramInfo for retrieving for retrieving the IL of a + * program + */ +#define CL_PROGRAM_IL_KHR 0x1169 + +extern CL_API_ENTRY cl_program CL_API_CALL +clCreateProgramWithILKHR(cl_context context, + const void * il, + size_t length, + cl_int * errcode_ret); + +typedef CL_API_ENTRY cl_program +(CL_API_CALL *clCreateProgramWithILKHR_fn)(cl_context context, + const void * il, + size_t length, + cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_2; + +/* Extension: cl_khr_image2d_from_buffer + * + * This extension allows a 2D image to be created from a cl_mem buffer without + * a copy. The type associated with a 2D image created from a buffer in an + * OpenCL program is image2d_t. Both the sampler and sampler-less read_image + * built-in functions are supported for 2D images and 2D images created from + * a buffer. Similarly, the write_image built-ins are also supported for 2D + * images created from a buffer. + * + * When the 2D image from buffer is created, the client must specify the + * width, height, image format (i.e. channel order and channel data type) + * and optionally the row pitch. + * + * The pitch specified must be a multiple of + * CL_DEVICE_IMAGE_PITCH_ALIGNMENT_KHR pixels. + * The base address of the buffer must be aligned to + * CL_DEVICE_IMAGE_BASE_ADDRESS_ALIGNMENT_KHR pixels. + */ + +#define CL_DEVICE_IMAGE_PITCH_ALIGNMENT_KHR 0x104A +#define CL_DEVICE_IMAGE_BASE_ADDRESS_ALIGNMENT_KHR 0x104B + + +/************************************** + * cl_khr_initialize_memory extension * + **************************************/ + #define CL_CONTEXT_MEMORY_INITIALIZE_KHR 0x2030 - - + + /************************************** * cl_khr_terminate_context extension * **************************************/ - + #define CL_DEVICE_TERMINATE_CAPABILITY_KHR 0x2031 #define CL_CONTEXT_TERMINATE_KHR 0x2032 #define cl_khr_terminate_context 1 -extern CL_API_ENTRY cl_int CL_API_CALL clTerminateContextKHR(cl_context /* context */) CL_EXT_SUFFIX__VERSION_1_2; +extern CL_API_ENTRY cl_int CL_API_CALL +clTerminateContextKHR(cl_context context) CL_EXT_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_int +(CL_API_CALL *clTerminateContextKHR_fn)(cl_context context) CL_EXT_SUFFIX__VERSION_1_2; + -typedef CL_API_ENTRY cl_int (CL_API_CALL *clTerminateContextKHR_fn)(cl_context /* context */) CL_EXT_SUFFIX__VERSION_1_2; - - /* * Extension: cl_khr_spir * - * This extension adds support to create an OpenCL program object from a + * This extension adds support to create an OpenCL program object from a * Standard Portable Intermediate Representation (SPIR) instance */ @@ -166,9 +203,30 @@ typedef CL_API_ENTRY cl_int (CL_API_CALL *clTerminateContextKHR_fn)(cl_context / #define CL_PROGRAM_BINARY_TYPE_INTERMEDIATE 0x40E1 +/***************************************** + * cl_khr_create_command_queue extension * + *****************************************/ +#define cl_khr_create_command_queue 1 + +typedef cl_bitfield cl_queue_properties_khr; + +extern CL_API_ENTRY cl_command_queue CL_API_CALL +clCreateCommandQueueWithPropertiesKHR(cl_context context, + cl_device_id device, + const cl_queue_properties_khr* properties, + cl_int* errcode_ret) CL_EXT_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_command_queue +(CL_API_CALL *clCreateCommandQueueWithPropertiesKHR_fn)(cl_context context, + cl_device_id device, + const cl_queue_properties_khr* properties, + cl_int* errcode_ret) CL_EXT_SUFFIX__VERSION_1_2; + + /****************************************** * cl_nv_device_attribute_query extension * ******************************************/ + /* cl_nv_device_attribute_query extension - no extension #define since it has no functions */ #define CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV 0x4000 #define CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV 0x4001 @@ -190,6 +248,7 @@ typedef CL_API_ENTRY cl_int (CL_API_CALL *clTerminateContextKHR_fn)(cl_context / /********************************* * cl_amd_device_attribute_query * *********************************/ + #define CL_DEVICE_PROFILING_TIMER_OFFSET_AMD 0x4036 #define CL_DEVICE_TOPOLOGY_AMD 0x4037 #define CL_DEVICE_BOARD_NAME_AMD 0x4038 @@ -382,14 +441,6 @@ typedef CL_API_ENTRY cl_mem #endif // -/*************************** -* cl_amd_command_intercept * -***************************/ -#define CL_CONTEXT_COMMAND_INTERCEPT_CALLBACK_AMD 0x403D -#define CL_QUEUE_COMMAND_INTERCEPT_ENABLE_AMD (1ull << 63) - -typedef cl_int (CL_CALLBACK * intercept_callback_fn)(cl_event, cl_int *); - /************************** * cl_amd_command_queue_info * **************************/ @@ -426,86 +477,120 @@ typedef CL_API_ENTRY cl_int (CL_API_CALL * clObjectSetValueForKeyAMD_fn)( void * /* value */) CL_API_SUFFIX__VERSION_1_1; // + /********************************* * cl_arm_printf extension *********************************/ + #define CL_PRINTF_CALLBACK_ARM 0x40B0 #define CL_PRINTF_BUFFERSIZE_ARM 0x40B1 -#ifdef CL_VERSION_1_1 - /*********************************** - * cl_ext_device_fission extension * - ***********************************/ - #define cl_ext_device_fission 1 - - extern CL_API_ENTRY cl_int CL_API_CALL - clReleaseDeviceEXT( cl_device_id /*device*/ ) CL_EXT_SUFFIX__VERSION_1_1; - - typedef CL_API_ENTRY cl_int - (CL_API_CALL *clReleaseDeviceEXT_fn)( cl_device_id /*device*/ ) CL_EXT_SUFFIX__VERSION_1_1; - extern CL_API_ENTRY cl_int CL_API_CALL - clRetainDeviceEXT( cl_device_id /*device*/ ) CL_EXT_SUFFIX__VERSION_1_1; - - typedef CL_API_ENTRY cl_int - (CL_API_CALL *clRetainDeviceEXT_fn)( cl_device_id /*device*/ ) CL_EXT_SUFFIX__VERSION_1_1; +/*********************************** +* cl_ext_device_fission extension +***********************************/ +#define cl_ext_device_fission 1 - typedef cl_ulong cl_device_partition_property_ext; - extern CL_API_ENTRY cl_int CL_API_CALL - clCreateSubDevicesEXT( cl_device_id /*in_device*/, - const cl_device_partition_property_ext * /* properties */, - cl_uint /*num_entries*/, - cl_device_id * /*out_devices*/, - cl_uint * /*num_devices*/ ) CL_EXT_SUFFIX__VERSION_1_1; +extern CL_API_ENTRY cl_int CL_API_CALL +clReleaseDeviceEXT(cl_device_id device) CL_EXT_SUFFIX__VERSION_1_1; - typedef CL_API_ENTRY cl_int - ( CL_API_CALL * clCreateSubDevicesEXT_fn)( cl_device_id /*in_device*/, - const cl_device_partition_property_ext * /* properties */, - cl_uint /*num_entries*/, - cl_device_id * /*out_devices*/, - cl_uint * /*num_devices*/ ) CL_EXT_SUFFIX__VERSION_1_1; +typedef CL_API_ENTRY cl_int +(CL_API_CALL *clReleaseDeviceEXT_fn)(cl_device_id device) CL_EXT_SUFFIX__VERSION_1_1; - /* cl_device_partition_property_ext */ - #define CL_DEVICE_PARTITION_EQUALLY_EXT 0x4050 - #define CL_DEVICE_PARTITION_BY_COUNTS_EXT 0x4051 - #define CL_DEVICE_PARTITION_BY_NAMES_EXT 0x4052 - #define CL_DEVICE_PARTITION_BY_AFFINITY_DOMAIN_EXT 0x4053 - - /* clDeviceGetInfo selectors */ - #define CL_DEVICE_PARENT_DEVICE_EXT 0x4054 - #define CL_DEVICE_PARTITION_TYPES_EXT 0x4055 - #define CL_DEVICE_AFFINITY_DOMAINS_EXT 0x4056 - #define CL_DEVICE_REFERENCE_COUNT_EXT 0x4057 - #define CL_DEVICE_PARTITION_STYLE_EXT 0x4058 - - /* clGetImageInfo enum */ - #define CL_IMAGE_BYTE_PITCH_AMD 0x4059 +extern CL_API_ENTRY cl_int CL_API_CALL +clRetainDeviceEXT(cl_device_id device) CL_EXT_SUFFIX__VERSION_1_1; + +typedef CL_API_ENTRY cl_int +(CL_API_CALL *clRetainDeviceEXT_fn)(cl_device_id device) CL_EXT_SUFFIX__VERSION_1_1; + +typedef cl_ulong cl_device_partition_property_ext; +extern CL_API_ENTRY cl_int CL_API_CALL +clCreateSubDevicesEXT(cl_device_id in_device, + const cl_device_partition_property_ext * properties, + cl_uint num_entries, + cl_device_id * out_devices, + cl_uint * num_devices) CL_EXT_SUFFIX__VERSION_1_1; + +typedef CL_API_ENTRY cl_int +(CL_API_CALL * clCreateSubDevicesEXT_fn)(cl_device_id in_device, + const cl_device_partition_property_ext * properties, + cl_uint num_entries, + cl_device_id * out_devices, + cl_uint * num_devices) CL_EXT_SUFFIX__VERSION_1_1; + +/* cl_device_partition_property_ext */ +#define CL_DEVICE_PARTITION_EQUALLY_EXT 0x4050 +#define CL_DEVICE_PARTITION_BY_COUNTS_EXT 0x4051 +#define CL_DEVICE_PARTITION_BY_NAMES_EXT 0x4052 +#define CL_DEVICE_PARTITION_BY_AFFINITY_DOMAIN_EXT 0x4053 + +/* clDeviceGetInfo selectors */ +#define CL_DEVICE_PARENT_DEVICE_EXT 0x4054 +#define CL_DEVICE_PARTITION_TYPES_EXT 0x4055 +#define CL_DEVICE_AFFINITY_DOMAINS_EXT 0x4056 +#define CL_DEVICE_REFERENCE_COUNT_EXT 0x4057 +#define CL_DEVICE_PARTITION_STYLE_EXT 0x4058 + +/* clGetImageInfo enum */ +#define CL_IMAGE_BYTE_PITCH_AMD 0x4059 + +/* error codes */ +#define CL_DEVICE_PARTITION_FAILED_EXT -1057 +#define CL_INVALID_PARTITION_COUNT_EXT -1058 +#define CL_INVALID_PARTITION_NAME_EXT -1059 + +/* CL_AFFINITY_DOMAINs */ +#define CL_AFFINITY_DOMAIN_L1_CACHE_EXT 0x1 +#define CL_AFFINITY_DOMAIN_L2_CACHE_EXT 0x2 +#define CL_AFFINITY_DOMAIN_L3_CACHE_EXT 0x3 +#define CL_AFFINITY_DOMAIN_L4_CACHE_EXT 0x4 +#define CL_AFFINITY_DOMAIN_NUMA_EXT 0x10 +#define CL_AFFINITY_DOMAIN_NEXT_FISSIONABLE_EXT 0x100 + +/* cl_device_partition_property_ext list terminators */ +#define CL_PROPERTIES_LIST_END_EXT ((cl_device_partition_property_ext) 0) +#define CL_PARTITION_BY_COUNTS_LIST_END_EXT ((cl_device_partition_property_ext) 0) +#define CL_PARTITION_BY_NAMES_LIST_END_EXT ((cl_device_partition_property_ext) 0 - 1) + + +/*********************************** + * cl_ext_migrate_memobject extension definitions + ***********************************/ +#define cl_ext_migrate_memobject 1 + +typedef cl_bitfield cl_mem_migration_flags_ext; + +#define CL_MIGRATE_MEM_OBJECT_HOST_EXT 0x1 + +#define CL_COMMAND_MIGRATE_MEM_OBJECT_EXT 0x4040 + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueMigrateMemObjectEXT(cl_command_queue command_queue, + cl_uint num_mem_objects, + const cl_mem * mem_objects, + cl_mem_migration_flags_ext flags, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event); + +typedef CL_API_ENTRY cl_int +(CL_API_CALL *clEnqueueMigrateMemObjectEXT_fn)(cl_command_queue command_queue, + cl_uint num_mem_objects, + const cl_mem * mem_objects, + cl_mem_migration_flags_ext flags, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event); - /* error codes */ - #define CL_DEVICE_PARTITION_FAILED_EXT -1057 - #define CL_INVALID_PARTITION_COUNT_EXT -1058 - #define CL_INVALID_PARTITION_NAME_EXT -1059 - - /* CL_AFFINITY_DOMAINs */ - #define CL_AFFINITY_DOMAIN_L1_CACHE_EXT 0x1 - #define CL_AFFINITY_DOMAIN_L2_CACHE_EXT 0x2 - #define CL_AFFINITY_DOMAIN_L3_CACHE_EXT 0x3 - #define CL_AFFINITY_DOMAIN_L4_CACHE_EXT 0x4 - #define CL_AFFINITY_DOMAIN_NUMA_EXT 0x10 - #define CL_AFFINITY_DOMAIN_NEXT_FISSIONABLE_EXT 0x100 - - /* cl_device_partition_property_ext list terminators */ - #define CL_PROPERTIES_LIST_END_EXT ((cl_device_partition_property_ext) 0) - #define CL_PARTITION_BY_COUNTS_LIST_END_EXT ((cl_device_partition_property_ext) 0) - #define CL_PARTITION_BY_NAMES_LIST_END_EXT ((cl_device_partition_property_ext) 0 - 1) /********************************* * cl_qcom_ext_host_ptr extension *********************************/ +#define cl_qcom_ext_host_ptr 1 #define CL_MEM_EXT_HOST_PTR_QCOM (1 << 29) -#define CL_DEVICE_EXT_MEM_PADDING_IN_BYTES_QCOM 0x40A0 +#define CL_DEVICE_EXT_MEM_PADDING_IN_BYTES_QCOM 0x40A0 #define CL_DEVICE_PAGE_SIZE_QCOM 0x40A1 #define CL_IMAGE_ROW_ALIGNMENT_QCOM 0x40A2 #define CL_IMAGE_SLICE_ALIGNMENT_QCOM 0x40A3 @@ -531,12 +616,21 @@ typedef struct _cl_mem_ext_host_ptr /* Type of external memory allocation. */ /* Legal values will be defined in layered extensions. */ cl_uint allocation_type; - + /* Host cache policy for this external memory allocation. */ cl_uint host_cache_policy; } cl_mem_ext_host_ptr; + +/******************************************* +* cl_qcom_ext_host_ptr_iocoherent extension +********************************************/ + +/* Cache policy specifying io-coherence */ +#define CL_MEM_HOST_IOCOHERENT_QCOM 0x40A9 + + /********************************* * cl_qcom_ion_host_ptr extension *********************************/ @@ -551,15 +645,30 @@ typedef struct _cl_mem_ion_host_ptr /* ION file descriptor */ int ion_filedesc; - + /* Host pointer to the ION allocated memory */ void* ion_hostptr; } cl_mem_ion_host_ptr; -#endif /* CL_VERSION_1_1 */ -#if defined(CL_VERSION_1_2) +/********************************* +* cl_qcom_android_native_buffer_host_ptr extension +*********************************/ + +#define CL_MEM_ANDROID_NATIVE_BUFFER_HOST_PTR_QCOM 0x40C6 + +typedef struct _cl_mem_android_native_buffer_host_ptr +{ + /* Type of external memory allocation. */ + /* Must be CL_MEM_ANDROID_NATIVE_BUFFER_HOST_PTR_QCOM for Android native buffers. */ + cl_mem_ext_host_ptr ext_host_ptr; + + /* Virtual pointer to the android native buffer */ + void* anb_ptr; + +} cl_mem_android_native_buffer_host_ptr; + /****************************************** * cl_img_yuv_image extension * @@ -569,20 +678,23 @@ typedef struct _cl_mem_ion_host_ptr #define CL_NV21_IMG 0x40D0 #define CL_YV12_IMG 0x40D1 + /****************************************** * cl_img_cached_allocations extension * ******************************************/ -/* Flag values used by clCreteBuffer */ -#define CL_MEM_USE_UNCACHED_CPU_MEMORY_IMG (1 << 26) -#define CL_MEM_USE_CACHED_CPU_MEMORY_IMG (1 << 27) +/* Flag values used by clCreateBuffer */ +#define CL_MEM_USE_UNCACHED_CPU_MEMORY_IMG (1 << 26) +#define CL_MEM_USE_CACHED_CPU_MEMORY_IMG (1 << 27) + /****************************************** * cl_img_use_gralloc_ptr extension * ******************************************/ +#define cl_img_use_gralloc_ptr 1 -/* Flag values used by clCreteBuffer */ -#define CL_MEM_USE_GRALLOC_PTR_IMG (1 << 28) +/* Flag values used by clCreateBuffer */ +#define CL_MEM_USE_GRALLOC_PTR_IMG (1 << 28) /* To be used by clGetEventInfo: */ #define CL_COMMAND_ACQUIRE_GRALLOC_OBJECTS_IMG 0x40D2 @@ -592,60 +704,75 @@ typedef struct _cl_mem_ion_host_ptr #define CL_GRALLOC_RESOURCE_NOT_ACQUIRED_IMG 0x40D4 extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueAcquireGrallocObjectsIMG(cl_command_queue /* command_queue */, - cl_uint /* num_objects */, - const cl_mem * /* mem_objects */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_2; +clEnqueueAcquireGrallocObjectsIMG(cl_command_queue command_queue, + cl_uint num_objects, + const cl_mem * mem_objects, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReleaseGrallocObjectsIMG(cl_command_queue /* command_queue */, - cl_uint /* num_objects */, - const cl_mem * /* mem_objects */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_2; +clEnqueueReleaseGrallocObjectsIMG(cl_command_queue command_queue, + cl_uint num_objects, + const cl_mem * mem_objects, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_2; -#endif /* CL_VERSION_1_2 */ -#ifdef CL_VERSION_2_0 /********************************* * cl_khr_subgroups extension *********************************/ #define cl_khr_subgroups 1 -/* cl_kernel_sub_group_info is declared in CL.h. */ +#if !defined(CL_VERSION_2_1) +/* For OpenCL 2.1 and newer, cl_kernel_sub_group_info is declared in CL.h. + In hindsight, there should have been a khr suffix on this type for + the extension, but keeping it un-suffixed to maintain backwards + compatibility. */ +typedef cl_uint cl_kernel_sub_group_info; +#endif /* cl_kernel_sub_group_info */ -#define CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE_KHR 0x2033 -#define CL_KERNEL_SUB_GROUP_COUNT_FOR_NDRANGE_KHR 0x2034 +#define CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE_KHR 0x2033 +#define CL_KERNEL_SUB_GROUP_COUNT_FOR_NDRANGE_KHR 0x2034 extern CL_API_ENTRY cl_int CL_API_CALL -clGetKernelSubGroupInfoKHR(cl_kernel /* in_kernel */, - cl_device_id /*in_device*/, - cl_kernel_sub_group_info /* param_name */, - size_t /*input_value_size*/, - const void * /*input_value*/, - size_t /*param_value_size*/, - void* /*param_value*/, - size_t* /*param_value_size_ret*/ ) CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED; - -typedef CL_API_ENTRY cl_int - ( CL_API_CALL * clGetKernelSubGroupInfoKHR_fn)(cl_kernel /* in_kernel */, - cl_device_id /*in_device*/, - cl_kernel_sub_group_info /* param_name */, - size_t /*input_value_size*/, - const void * /*input_value*/, - size_t /*param_value_size*/, - void* /*param_value*/, - size_t* /*param_value_size_ret*/ ) CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED; -#endif /* CL_VERSION_2_0 */ +clGetKernelSubGroupInfoKHR(cl_kernel in_kernel, + cl_device_id in_device, + cl_kernel_sub_group_info param_name, + size_t input_value_size, + const void * input_value, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED; + +typedef CL_API_ENTRY cl_int +(CL_API_CALL * clGetKernelSubGroupInfoKHR_fn)(cl_kernel in_kernel, + cl_device_id in_device, + cl_kernel_sub_group_info param_name, + size_t input_value_size, + const void * input_value, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED; + + +/********************************* +* cl_khr_mipmap_image extension +*********************************/ + +/* cl_sampler_properties */ +#define CL_SAMPLER_MIP_FILTER_MODE_KHR 0x1155 +#define CL_SAMPLER_LOD_MIN_KHR 0x1156 +#define CL_SAMPLER_LOD_MAX_KHR 0x1157 + -#ifdef CL_VERSION_2_1 /********************************* * cl_khr_priority_hints extension *********************************/ +/* This extension define is for backwards compatibility. + It shouldn't be required since this extension has no new functions. */ #define cl_khr_priority_hints 1 typedef cl_uint cl_queue_priority_khr; @@ -658,12 +785,12 @@ typedef cl_uint cl_queue_priority_khr; #define CL_QUEUE_PRIORITY_MED_KHR (1<<1) #define CL_QUEUE_PRIORITY_LOW_KHR (1<<2) -#endif /* CL_VERSION_2_1 */ -#ifdef CL_VERSION_2_1 /********************************* * cl_khr_throttle_hints extension *********************************/ +/* This extension define is for backwards compatibility. + It shouldn't be required since this extension has no new functions. */ #define cl_khr_throttle_hints 1 typedef cl_uint cl_queue_throttle_khr; @@ -676,24 +803,65 @@ typedef cl_uint cl_queue_throttle_khr; #define CL_QUEUE_THROTTLE_MED_KHR (1<<1) #define CL_QUEUE_THROTTLE_LOW_KHR (1<<2) -#endif /* CL_VERSION_2_1 */ -#ifdef CL_VERSION_2_2 /********************************* * cl_khr_subgroup_named_barrier *********************************/ +/* This extension define is for backwards compatibility. + It shouldn't be required since this extension has no new functions. */ #define cl_khr_subgroup_named_barrier 1 /* cl_device_info */ #define CL_DEVICE_MAX_NAMED_BARRIER_COUNT_KHR 0x2035 -#endif /* CL_VERSION_2_2 */ + +/********************************* +* cl_khr_extended_versioning +*********************************/ + +#define CL_VERSION_MAJOR_BITS_KHR (10) +#define CL_VERSION_MINOR_BITS_KHR (10) +#define CL_VERSION_PATCH_BITS_KHR (12) + +#define CL_VERSION_MAJOR_MASK_KHR ((1 << CL_VERSION_MAJOR_BITS_KHR) - 1) +#define CL_VERSION_MINOR_MASK_KHR ((1 << CL_VERSION_MINOR_BITS_KHR) - 1) +#define CL_VERSION_PATCH_MASK_KHR ((1 << CL_VERSION_PATCH_BITS_KHR) - 1) + +#define CL_VERSION_MAJOR_KHR(version) ((version) >> (CL_VERSION_MINOR_BITS_KHR + CL_VERSION_PATCH_BITS_KHR)) +#define CL_VERSION_MINOR_KHR(version) (((version) >> CL_VERSION_PATCH_BITS_KHR) & CL_VERSION_MINOR_MASK_KHR) +#define CL_VERSION_PATCH_KHR(version) ((version) & CL_VERSION_PATCH_MASK_KHR) + +#define CL_MAKE_VERSION_KHR(major, minor, patch) \ + ((((major) & CL_VERSION_MAJOR_MASK_KHR) << (CL_VERSION_MINOR_BITS_KHR + CL_VERSION_PATCH_BITS_KHR)) | \ + (((minor) & CL_VERSION_MINOR_MASK_KHR) << CL_VERSION_PATCH_BITS_KHR) | \ + ((patch) & CL_VERSION_PATCH_MASK_KHR)) + +typedef cl_uint cl_version_khr; + +#define CL_NAME_VERSION_MAX_NAME_SIZE_KHR 64 + +typedef struct _cl_name_version_khr +{ + cl_version_khr version; + char name[CL_NAME_VERSION_MAX_NAME_SIZE_KHR]; +} cl_name_version_khr; + +/* cl_platform_info */ +#define CL_PLATFORM_NUMERIC_VERSION_KHR 0x0906 +#define CL_PLATFORM_EXTENSIONS_WITH_VERSION_KHR 0x0907 + +/* cl_device_info */ +#define CL_DEVICE_NUMERIC_VERSION_KHR 0x105E +#define CL_DEVICE_OPENCL_C_NUMERIC_VERSION_KHR 0x105F +#define CL_DEVICE_EXTENSIONS_WITH_VERSION_KHR 0x1060 +#define CL_DEVICE_ILS_WITH_VERSION_KHR 0x1061 +#define CL_DEVICE_BUILT_IN_KERNELS_WITH_VERSION_KHR 0x1062 + /********************************** * cl_arm_import_memory extension * **********************************/ - -#ifdef CL_VERSION_1_0 +#define cl_arm_import_memory 1 typedef intptr_t cl_import_properties_arm; @@ -706,8 +874,17 @@ typedef intptr_t cl_import_properties_arm; /* DMA BUF memory type value for CL_IMPORT_TYPE_ARM property */ #define CL_IMPORT_TYPE_DMA_BUF_ARM 0x40B4 -/* Secure DMA BUF memory type value for CL_IMPORT_TYPE_ARM property */ -#define CL_IMPORT_TYPE_SECURE_ARM 0x40B5 +/* Protected memory property */ +#define CL_IMPORT_TYPE_PROTECTED_ARM 0x40B5 + +/* Android hardware buffer type value for CL_IMPORT_TYPE_ARM property */ +#define CL_IMPORT_TYPE_ANDROID_HARDWARE_BUFFER_ARM 0x41E2 + +/* Data consistency with host property */ +#define CL_IMPORT_DMA_BUF_DATA_CONSISTENCY_WITH_HOST_ARM 0x41E3 + +/* Import memory size value to indicate a size for the whole buffer */ +#define CL_IMPORT_MEMORY_WHOLE_ALLOCATION_ARM SIZE_MAX /* This extension adds a new function that allows for direct memory import into * OpenCL via the clImportMemoryARM function. @@ -734,13 +911,10 @@ clImportMemoryARM( cl_context context, cl_int *errcode_ret) CL_EXT_SUFFIX__VERSION_1_0; -#endif /* CL_VERSION_1_0 */ - /****************************************** * cl_arm_shared_virtual_memory extension * ******************************************/ - -#ifdef CL_VERSION_1_2 +#define cl_arm_shared_virtual_memory 1 /* Used by clGetDeviceInfo */ #define CL_DEVICE_SVM_CAPABILITIES_ARM 0x40B6 @@ -774,76 +948,100 @@ typedef cl_uint cl_kernel_exec_info_arm; typedef cl_bitfield cl_device_svm_capabilities_arm; extern CL_API_ENTRY void * CL_API_CALL -clSVMAllocARM(cl_context /* context */, - cl_svm_mem_flags_arm /* flags */, - size_t /* size */, - cl_uint /* alignment */) CL_EXT_SUFFIX__VERSION_1_2; +clSVMAllocARM(cl_context context, + cl_svm_mem_flags_arm flags, + size_t size, + cl_uint alignment) CL_EXT_SUFFIX__VERSION_1_2; extern CL_API_ENTRY void CL_API_CALL -clSVMFreeARM(cl_context /* context */, - void * /* svm_pointer */) CL_EXT_SUFFIX__VERSION_1_2; +clSVMFreeARM(cl_context context, + void * svm_pointer) CL_EXT_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMFreeARM(cl_command_queue /* command_queue */, - cl_uint /* num_svm_pointers */, - void *[] /* svm_pointers[] */, - void (CL_CALLBACK * /*pfn_free_func*/)(cl_command_queue /* queue */, - cl_uint /* num_svm_pointers */, - void *[] /* svm_pointers[] */, - void * /* user_data */), - void * /* user_data */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_2; +clEnqueueSVMFreeARM(cl_command_queue command_queue, + cl_uint num_svm_pointers, + void * svm_pointers[], + void (CL_CALLBACK * pfn_free_func)(cl_command_queue queue, + cl_uint num_svm_pointers, + void * svm_pointers[], + void * user_data), + void * user_data, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMMemcpyARM(cl_command_queue /* command_queue */, - cl_bool /* blocking_copy */, - void * /* dst_ptr */, - const void * /* src_ptr */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_2; +clEnqueueSVMMemcpyARM(cl_command_queue command_queue, + cl_bool blocking_copy, + void * dst_ptr, + const void * src_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMMemFillARM(cl_command_queue /* command_queue */, - void * /* svm_ptr */, - const void * /* pattern */, - size_t /* pattern_size */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_2; +clEnqueueSVMMemFillARM(cl_command_queue command_queue, + void * svm_ptr, + const void * pattern, + size_t pattern_size, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMMapARM(cl_command_queue /* command_queue */, - cl_bool /* blocking_map */, - cl_map_flags /* flags */, - void * /* svm_ptr */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_2; +clEnqueueSVMMapARM(cl_command_queue command_queue, + cl_bool blocking_map, + cl_map_flags flags, + void * svm_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueSVMUnmapARM(cl_command_queue /* command_queue */, - void * /* svm_ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_EXT_SUFFIX__VERSION_1_2; +clEnqueueSVMUnmapARM(cl_command_queue command_queue, + void * svm_ptr, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_EXT_SUFFIX__VERSION_1_2; extern CL_API_ENTRY cl_int CL_API_CALL -clSetKernelArgSVMPointerARM(cl_kernel /* kernel */, - cl_uint /* arg_index */, - const void * /* arg_value */) CL_EXT_SUFFIX__VERSION_1_2; -extern CL_API_ENTRY cl_int CL_API_CALL -clSetKernelExecInfoARM(cl_kernel /* kernel */, - cl_kernel_exec_info_arm /* param_name */, - size_t /* param_value_size */, - const void * /* param_value */) CL_EXT_SUFFIX__VERSION_1_2; +clSetKernelArgSVMPointerARM(cl_kernel kernel, + cl_uint arg_index, + const void * arg_value) CL_EXT_SUFFIX__VERSION_1_2; -#endif /* CL_VERSION_1_2 */ +extern CL_API_ENTRY cl_int CL_API_CALL +clSetKernelExecInfoARM(cl_kernel kernel, + cl_kernel_exec_info_arm param_name, + size_t param_value_size, + const void * param_value) CL_EXT_SUFFIX__VERSION_1_2; + +/******************************** + * cl_arm_get_core_id extension * + ********************************/ + +#ifdef CL_VERSION_1_2 + +#define cl_arm_get_core_id 1 + +/* Device info property for bitfield of cores present */ +#define CL_DEVICE_COMPUTE_UNITS_BITFIELD_ARM 0x40BF + +#endif /* CL_VERSION_1_2 */ + +/********************************* +* cl_arm_job_slot_selection +*********************************/ + +#define cl_arm_job_slot_selection 1 + +/* cl_device_info */ +#define CL_DEVICE_JOB_SLOTS_ARM 0x41E0 + +/* cl_command_queue_properties */ +#define CL_QUEUE_JOB_SLOT_ARM 0x41E1 #ifdef __cplusplus } diff --git a/amdocl/CL/cl_gl.h b/amdocl/CL/cl_gl.h index 945daa83d7..fbdaf62977 100644 --- a/amdocl/CL/cl_gl.h +++ b/amdocl/CL/cl_gl.h @@ -1,5 +1,5 @@ /********************************************************************************** - * Copyright (c) 2008-2015 The Khronos Group Inc. + * Copyright (c) 2008-2019 The Khronos Group Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and/or associated documentation files (the @@ -29,11 +29,7 @@ #ifndef __OPENCL_CL_GL_H #define __OPENCL_CL_GL_H -#ifdef __APPLE__ -#include -#else #include -#endif #ifdef __cplusplus extern "C" { @@ -49,110 +45,118 @@ typedef struct __GLsync *cl_GLsync; #define CL_GL_OBJECT_TEXTURE2D 0x2001 #define CL_GL_OBJECT_TEXTURE3D 0x2002 #define CL_GL_OBJECT_RENDERBUFFER 0x2003 +#ifdef CL_VERSION_1_2 #define CL_GL_OBJECT_TEXTURE2D_ARRAY 0x200E #define CL_GL_OBJECT_TEXTURE1D 0x200F #define CL_GL_OBJECT_TEXTURE1D_ARRAY 0x2010 #define CL_GL_OBJECT_TEXTURE_BUFFER 0x2011 +#endif /* cl_gl_texture_info */ #define CL_GL_TEXTURE_TARGET 0x2004 #define CL_GL_MIPMAP_LEVEL 0x2005 +#ifdef CL_VERSION_1_2 #define CL_GL_NUM_SAMPLES 0x2012 +#endif extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromGLBuffer(cl_context /* context */, - cl_mem_flags /* flags */, - cl_GLuint /* bufobj */, - int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateFromGLBuffer(cl_context context, + cl_mem_flags flags, + cl_GLuint bufobj, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromGLTexture(cl_context /* context */, - cl_mem_flags /* flags */, - cl_GLenum /* target */, - cl_GLint /* miplevel */, - cl_GLuint /* texture */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_2; - +clCreateFromGLTexture(cl_context context, + cl_mem_flags flags, + cl_GLenum target, + cl_GLint miplevel, + cl_GLuint texture, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +#endif + extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromGLRenderbuffer(cl_context /* context */, - cl_mem_flags /* flags */, - cl_GLuint /* renderbuffer */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_0; +clCreateFromGLRenderbuffer(cl_context context, + cl_mem_flags flags, + cl_GLuint renderbuffer, + cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clGetGLObjectInfo(cl_mem /* memobj */, - cl_gl_object_type * /* gl_object_type */, - cl_GLuint * /* gl_object_name */) CL_API_SUFFIX__VERSION_1_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clGetGLTextureInfo(cl_mem /* memobj */, - cl_gl_texture_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; +clGetGLObjectInfo(cl_mem memobj, + cl_gl_object_type * gl_object_type, + cl_GLuint * gl_object_name) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueAcquireGLObjects(cl_command_queue /* command_queue */, - cl_uint /* num_objects */, - const cl_mem * /* mem_objects */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clGetGLTextureInfo(cl_mem memobj, + cl_gl_texture_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReleaseGLObjects(cl_command_queue /* command_queue */, - cl_uint /* num_objects */, - const cl_mem * /* mem_objects */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_1_0; +clEnqueueAcquireGLObjects(cl_command_queue command_queue, + cl_uint num_objects, + const cl_mem * mem_objects, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; + +extern CL_API_ENTRY cl_int CL_API_CALL +clEnqueueReleaseGLObjects(cl_command_queue command_queue, + cl_uint num_objects, + const cl_mem * mem_objects, + cl_uint num_events_in_wait_list, + const cl_event * event_wait_list, + cl_event * event) CL_API_SUFFIX__VERSION_1_0; /* Deprecated OpenCL 1.1 APIs */ extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED cl_mem CL_API_CALL -clCreateFromGLTexture2D(cl_context /* context */, - cl_mem_flags /* flags */, - cl_GLenum /* target */, - cl_GLint /* miplevel */, - cl_GLuint /* texture */, - cl_int * /* errcode_ret */) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - +clCreateFromGLTexture2D(cl_context context, + cl_mem_flags flags, + cl_GLenum target, + cl_GLint miplevel, + cl_GLuint texture, + cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + extern CL_API_ENTRY CL_EXT_PREFIX__VERSION_1_1_DEPRECATED cl_mem CL_API_CALL -clCreateFromGLTexture3D(cl_context /* context */, - cl_mem_flags /* flags */, - cl_GLenum /* target */, - cl_GLint /* miplevel */, - cl_GLuint /* texture */, - cl_int * /* errcode_ret */) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - +clCreateFromGLTexture3D(cl_context context, + cl_mem_flags flags, + cl_GLenum target, + cl_GLint miplevel, + cl_GLuint texture, + cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + /* cl_khr_gl_sharing extension */ - + #define cl_khr_gl_sharing 1 - + typedef cl_uint cl_gl_context_info; - + /* Additional Error Codes */ #define CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR -1000 - + /* cl_gl_context_info */ #define CL_CURRENT_DEVICE_FOR_GL_CONTEXT_KHR 0x2006 #define CL_DEVICES_FOR_GL_CONTEXT_KHR 0x2007 - + /* Additional cl_context_properties */ #define CL_GL_CONTEXT_KHR 0x2008 #define CL_EGL_DISPLAY_KHR 0x2009 #define CL_GLX_DISPLAY_KHR 0x200A #define CL_WGL_HDC_KHR 0x200B #define CL_CGL_SHAREGROUP_KHR 0x200C - + extern CL_API_ENTRY cl_int CL_API_CALL -clGetGLContextInfoKHR(const cl_context_properties * /* properties */, - cl_gl_context_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_1_0; - +clGetGLContextInfoKHR(const cl_context_properties * properties, + cl_gl_context_info param_name, + size_t param_value_size, + void * param_value, + size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + typedef CL_API_ENTRY cl_int (CL_API_CALL *clGetGLContextInfoKHR_fn)( const cl_context_properties * properties, cl_gl_context_info param_name, diff --git a/amdocl/CL/cl_gl_ext.h b/amdocl/CL/cl_gl_ext.h index e3c14c6408..c26d31abed 100644 --- a/amdocl/CL/cl_gl_ext.h +++ b/amdocl/CL/cl_gl_ext.h @@ -1,5 +1,5 @@ /********************************************************************************** - * Copyright (c) 2008-2015 The Khronos Group Inc. + * Copyright (c) 2008-2019 The Khronos Group Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and/or associated documentation files (the @@ -26,11 +26,6 @@ * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. **********************************************************************************/ -/* $Revision: 11708 $ on $Date: 2010-06-13 23:36:24 -0700 (Sun, 13 Jun 2010) $ */ - -/* cl_gl_ext.h contains vendor (non-KHR) OpenCL extensions which have */ -/* OpenGL dependencies. */ - #ifndef __OPENCL_CL_GL_EXT_H #define __OPENCL_CL_GL_EXT_H @@ -38,34 +33,17 @@ extern "C" { #endif -#ifdef __APPLE__ - #include -#else - #include -#endif - -/* - * For each extension, follow this template - * cl_VEN_extname extension */ -/* #define cl_VEN_extname 1 - * ... define new types, if any - * ... define new tokens, if any - * ... define new APIs, if any - * - * If you need GLtypes here, mirror them with a cl_GLtype, rather than including a GL header - * This allows us to avoid having to decide whether to include GL headers or GLES here. - */ +#include /* - * cl_khr_gl_event extension - * See section 9.9 in the OpenCL 1.1 spec for more information + * cl_khr_gl_event extension */ #define CL_COMMAND_GL_FENCE_SYNC_OBJECT_KHR 0x200D extern CL_API_ENTRY cl_event CL_API_CALL -clCreateEventFromGLsyncKHR(cl_context /* context */, - cl_GLsync /* cl_GLsync */, - cl_int * /* errcode_ret */) CL_EXT_SUFFIX__VERSION_1_1; +clCreateEventFromGLsyncKHR(cl_context context, + cl_GLsync cl_GLsync, + cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_1; #ifdef __cplusplus } diff --git a/amdocl/CL/cl_icd.h b/amdocl/CL/cl_icd.h new file mode 100644 index 0000000000..2be64719b6 --- /dev/null +++ b/amdocl/CL/cl_icd.h @@ -0,0 +1,1269 @@ +/******************************************************************************* + * Copyright (c) 2019 The Khronos Group Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and/or associated documentation files (the + * "Materials"), to deal in the Materials without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Materials, and to + * permit persons to whom the Materials are furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Materials. + * + * MODIFICATIONS TO THIS FILE MAY MEAN IT NO LONGER ACCURATELY REFLECTS + * KHRONOS STANDARDS. THE UNMODIFIED, NORMATIVE VERSIONS OF KHRONOS + * SPECIFICATIONS AND HEADER INFORMATION ARE LOCATED AT + * https://www.khronos.org/registry/ + * + * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. + ******************************************************************************/ + +#ifndef OPENCL_CL_ICD_H +#define OPENCL_CL_ICD_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This file contains pointer type definitions for each of the CL API calls as + * well as a type definition for the dispatch table used by the Khronos ICD + * loader (see cl_khr_icd extension specification for background). + */ + +/* API function pointer definitions */ + +// Platform APIs +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetPlatformIDs)( + cl_uint num_entries, cl_platform_id *platforms, + cl_uint *num_platforms) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetPlatformInfo)( + cl_platform_id platform, cl_platform_info param_name, + size_t param_value_size, void *param_value, + size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +// Device APIs +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetDeviceIDs)( + cl_platform_id platform, cl_device_type device_type, cl_uint num_entries, + cl_device_id *devices, cl_uint *num_devices) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetDeviceInfo)( + cl_device_id device, cl_device_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clCreateSubDevices)( + cl_device_id in_device, + const cl_device_partition_property *partition_properties, + cl_uint num_entries, cl_device_id *out_devices, cl_uint *num_devices); + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainDevice)( + cl_device_id device) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseDevice)( + cl_device_id device) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clCreateSubDevices; +typedef void *cl_api_clRetainDevice; +typedef void *cl_api_clReleaseDevice; + +#endif + +// Context APIs +typedef CL_API_ENTRY cl_context(CL_API_CALL *cl_api_clCreateContext)( + const cl_context_properties *properties, cl_uint num_devices, + const cl_device_id *devices, + void(CL_CALLBACK *pfn_notify)(const char *, const void *, size_t, void *), + void *user_data, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_context(CL_API_CALL *cl_api_clCreateContextFromType)( + const cl_context_properties *properties, cl_device_type device_type, + void(CL_CALLBACK *pfn_notify)(const char *, const void *, size_t, void *), + void *user_data, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainContext)( + cl_context context) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseContext)( + cl_context context) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetContextInfo)( + cl_context context, cl_context_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +// Command Queue APIs +typedef CL_API_ENTRY cl_command_queue(CL_API_CALL *cl_api_clCreateCommandQueue)( + cl_context context, cl_device_id device, + cl_command_queue_properties properties, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_2_0 + +typedef CL_API_ENTRY +cl_command_queue(CL_API_CALL *cl_api_clCreateCommandQueueWithProperties)( + cl_context /* context */, cl_device_id /* device */, + const cl_queue_properties * /* properties */, + cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; + +#else + +typedef void *cl_api_clCreateCommandQueueWithProperties; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainCommandQueue)( + cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseCommandQueue)( + cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetCommandQueueInfo)( + cl_command_queue command_queue, cl_command_queue_info param_name, + size_t param_value_size, void *param_value, + size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +// Memory Object APIs +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateBuffer)( + cl_context context, cl_mem_flags flags, size_t size, void *host_ptr, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateImage)( + cl_context context, cl_mem_flags flags, const cl_image_format *image_format, + const cl_image_desc *image_desc, void *host_ptr, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clCreateImage; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainMemObject)( + cl_mem memobj) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseMemObject)( + cl_mem memobj) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetSupportedImageFormats)( + cl_context context, cl_mem_flags flags, cl_mem_object_type image_type, + cl_uint num_entries, cl_image_format *image_formats, + cl_uint *num_image_formats) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetMemObjectInfo)( + cl_mem memobj, cl_mem_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetImageInfo)( + cl_mem image, cl_image_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_2_0 + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreatePipe)( + cl_context /* context */, cl_mem_flags /* flags */, + cl_uint /* pipe_packet_size */, cl_uint /* pipe_max_packets */, + const cl_pipe_properties * /* properties */, + cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetPipeInfo)( + cl_mem /* pipe */, cl_pipe_info /* param_name */, + size_t /* param_value_size */, void * /* param_value */, + size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY void *(CL_API_CALL *cl_api_clSVMAlloc)( + cl_context /* context */, cl_svm_mem_flags /* flags */, size_t /* size */, + unsigned int /* alignment */)CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY void(CL_API_CALL *cl_api_clSVMFree)( + cl_context /* context */, + void * /* svm_pointer */) CL_API_SUFFIX__VERSION_2_0; + +#else + +typedef void *cl_api_clCreatePipe; +typedef void *cl_api_clGetPipeInfo; +typedef void *cl_api_clSVMAlloc; +typedef void *cl_api_clSVMFree; + +#endif + +// Sampler APIs +typedef CL_API_ENTRY cl_sampler(CL_API_CALL *cl_api_clCreateSampler)( + cl_context context, cl_bool normalized_coords, + cl_addressing_mode addressing_mode, cl_filter_mode filter_mode, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainSampler)( + cl_sampler sampler) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseSampler)( + cl_sampler sampler) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetSamplerInfo)( + cl_sampler sampler, cl_sampler_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_2_0 + +typedef CL_API_ENTRY +cl_sampler(CL_API_CALL *cl_api_clCreateSamplerWithProperties)( + cl_context /* context */, + const cl_sampler_properties * /* sampler_properties */, + cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; + +#else + +typedef void *cl_api_clCreateSamplerWithProperties; + +#endif + +// Program Object APIs +typedef CL_API_ENTRY cl_program(CL_API_CALL *cl_api_clCreateProgramWithSource)( + cl_context context, cl_uint count, const char **strings, + const size_t *lengths, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_program(CL_API_CALL *cl_api_clCreateProgramWithBinary)( + cl_context context, cl_uint num_devices, const cl_device_id *device_list, + const size_t *lengths, const unsigned char **binaries, + cl_int *binary_status, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY +cl_program(CL_API_CALL *cl_api_clCreateProgramWithBuiltInKernels)( + cl_context context, cl_uint num_devices, const cl_device_id *device_list, + const char *kernel_names, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clCreateProgramWithBuiltInKernels; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainProgram)( + cl_program program) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseProgram)( + cl_program program) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clBuildProgram)( + cl_program program, cl_uint num_devices, const cl_device_id *device_list, + const char *options, + void(CL_CALLBACK *pfn_notify)(cl_program program, void *user_data), + void *user_data) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clCompileProgram)( + cl_program program, cl_uint num_devices, const cl_device_id *device_list, + const char *options, cl_uint num_input_headers, + const cl_program *input_headers, const char **header_include_names, + void(CL_CALLBACK *pfn_notify)(cl_program program, void *user_data), + void *user_data) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_program(CL_API_CALL *cl_api_clLinkProgram)( + cl_context context, cl_uint num_devices, const cl_device_id *device_list, + const char *options, cl_uint num_input_programs, + const cl_program *input_programs, + void(CL_CALLBACK *pfn_notify)(cl_program program, void *user_data), + void *user_data, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clCompileProgram; +typedef void *cl_api_clLinkProgram; + +#endif + +#ifdef CL_VERSION_2_2 + +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clSetProgramSpecializationConstant)( + cl_program program, cl_uint spec_id, size_t spec_size, + const void *spec_value) CL_API_SUFFIX__VERSION_2_2; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clSetProgramReleaseCallback)( + cl_program program, + void(CL_CALLBACK *pfn_notify)(cl_program program, void *user_data), + void *user_data) CL_API_SUFFIX__VERSION_2_2; + +#else + +typedef void *cl_api_clSetProgramSpecializationConstant; +typedef void *cl_api_clSetProgramReleaseCallback; + +#endif + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clUnloadPlatformCompiler)( + cl_platform_id platform) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clUnloadPlatformCompiler; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetProgramInfo)( + cl_program program, cl_program_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetProgramBuildInfo)( + cl_program program, cl_device_id device, cl_program_build_info param_name, + size_t param_value_size, void *param_value, + size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +// Kernel Object APIs +typedef CL_API_ENTRY cl_kernel(CL_API_CALL *cl_api_clCreateKernel)( + cl_program program, const char *kernel_name, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clCreateKernelsInProgram)( + cl_program program, cl_uint num_kernels, cl_kernel *kernels, + cl_uint *num_kernels_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainKernel)( + cl_kernel kernel) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseKernel)( + cl_kernel kernel) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clSetKernelArg)( + cl_kernel kernel, cl_uint arg_index, size_t arg_size, + const void *arg_value) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetKernelInfo)( + cl_kernel kernel, cl_kernel_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetKernelArgInfo)( + cl_kernel kernel, cl_uint arg_indx, cl_kernel_arg_info param_name, + size_t param_value_size, void *param_value, + size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clGetKernelArgInfo; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetKernelWorkGroupInfo)( + cl_kernel kernel, cl_device_id device, cl_kernel_work_group_info param_name, + size_t param_value_size, void *param_value, + size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_2_0 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clSetKernelArgSVMPointer)( + cl_kernel /* kernel */, cl_uint /* arg_index */, + const void * /* arg_value */) CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clSetKernelExecInfo)( + cl_kernel /* kernel */, cl_kernel_exec_info /* param_name */, + size_t /* param_value_size */, + const void * /* param_value */) CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetKernelSubGroupInfoKHR)( + cl_kernel /* in_kernel */, cl_device_id /*in_device*/, + cl_kernel_sub_group_info /* param_name */, size_t /*input_value_size*/, + const void * /*input_value*/, size_t /*param_value_size*/, + void * /*param_value*/, + size_t * /*param_value_size_ret*/) CL_EXT_SUFFIX__VERSION_2_0; + +#else + +typedef void *cl_api_clSetKernelArgSVMPointer; +typedef void *cl_api_clSetKernelExecInfo; +typedef void *cl_api_clGetKernelSubGroupInfoKHR; + +#endif + +// Event Object APIs +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clWaitForEvents)( + cl_uint num_events, const cl_event *event_list) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetEventInfo)( + cl_event event, cl_event_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainEvent)(cl_event event) + CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseEvent)(cl_event event) + CL_API_SUFFIX__VERSION_1_0; + +// Profiling APIs +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetEventProfilingInfo)( + cl_event event, cl_profiling_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +// Flush and Finish APIs +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clFlush)( + cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clFinish)( + cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; + +// Enqueued Commands APIs +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueReadBuffer)( + cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_read, + size_t offset, size_t cb, void *ptr, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueReadBufferRect)( + cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_read, + const size_t *buffer_origin, const size_t *host_origin, + const size_t *region, size_t buffer_row_pitch, size_t buffer_slice_pitch, + size_t host_row_pitch, size_t host_slice_pitch, void *ptr, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_1; + +#else + +typedef void *cl_api_clEnqueueReadBufferRect; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueWriteBuffer)( + cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, + size_t offset, size_t cb, const void *ptr, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueWriteBufferRect)( + cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_read, + const size_t *buffer_origin, const size_t *host_origin, + const size_t *region, size_t buffer_row_pitch, size_t buffer_slice_pitch, + size_t host_row_pitch, size_t host_slice_pitch, const void *ptr, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_1; + +#else + +typedef void *cl_api_clEnqueueWriteBufferRect; + +#endif + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueFillBuffer)( + cl_command_queue command_queue, cl_mem buffer, const void *pattern, + size_t pattern_size, size_t offset, size_t cb, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clEnqueueFillBuffer; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueCopyBuffer)( + cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_buffer, + size_t src_offset, size_t dst_offset, size_t cb, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_1 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueCopyBufferRect)( + cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_buffer, + const size_t *src_origin, const size_t *dst_origin, const size_t *region, + size_t src_row_pitch, size_t src_slice_pitch, size_t dst_row_pitch, + size_t dst_slice_pitch, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_1; + +#else + +typedef void *cl_api_clEnqueueCopyBufferRect; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueReadImage)( + cl_command_queue command_queue, cl_mem image, cl_bool blocking_read, + const size_t *origin, const size_t *region, size_t row_pitch, + size_t slice_pitch, void *ptr, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueWriteImage)( + cl_command_queue command_queue, cl_mem image, cl_bool blocking_write, + const size_t *origin, const size_t *region, size_t input_row_pitch, + size_t input_slice_pitch, const void *ptr, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueFillImage)( + cl_command_queue command_queue, cl_mem image, const void *fill_color, + const size_t origin[3], const size_t region[3], + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clEnqueueFillImage; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueCopyImage)( + cl_command_queue command_queue, cl_mem src_image, cl_mem dst_image, + const size_t *src_origin, const size_t *dst_origin, const size_t *region, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueCopyImageToBuffer)( + cl_command_queue command_queue, cl_mem src_image, cl_mem dst_buffer, + const size_t *src_origin, const size_t *region, size_t dst_offset, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueCopyBufferToImage)( + cl_command_queue command_queue, cl_mem src_buffer, cl_mem dst_image, + size_t src_offset, const size_t *dst_origin, const size_t *region, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY void *(CL_API_CALL *cl_api_clEnqueueMapBuffer)( + cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_map, + cl_map_flags map_flags, size_t offset, size_t cb, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event, cl_int *errcode_ret)CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY void *(CL_API_CALL *cl_api_clEnqueueMapImage)( + cl_command_queue command_queue, cl_mem image, cl_bool blocking_map, + cl_map_flags map_flags, const size_t *origin, const size_t *region, + size_t *image_row_pitch, size_t *image_slice_pitch, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event, cl_int *errcode_ret)CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueUnmapMemObject)( + cl_command_queue command_queue, cl_mem memobj, void *mapped_ptr, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueMigrateMemObjects)( + cl_command_queue command_queue, cl_uint num_mem_objects, + const cl_mem *mem_objects, cl_mem_migration_flags flags, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clEnqueueMigrateMemObjects; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueNDRangeKernel)( + cl_command_queue command_queue, cl_kernel kernel, cl_uint work_dim, + const size_t *global_work_offset, const size_t *global_work_size, + const size_t *local_work_size, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueTask)( + cl_command_queue command_queue, cl_kernel kernel, + cl_uint num_events_in_wait_list, const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueNativeKernel)( + cl_command_queue command_queue, void(CL_CALLBACK *user_func)(void *), + void *args, size_t cb_args, cl_uint num_mem_objects, const cl_mem *mem_list, + const void **args_mem_loc, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +#ifdef CL_VERSION_1_2 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueMarkerWithWaitList)( + cl_command_queue command_queue, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueBarrierWithWaitList)( + cl_command_queue command_queue, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY void *( + CL_API_CALL *cl_api_clGetExtensionFunctionAddressForPlatform)( + cl_platform_id platform, + const char *function_name)CL_API_SUFFIX__VERSION_1_2; + +#else + +typedef void *cl_api_clEnqueueMarkerWithWaitList; +typedef void *cl_api_clEnqueueBarrierWithWaitList; +typedef void *cl_api_clGetExtensionFunctionAddressForPlatform; + +#endif + +// Shared Virtual Memory APIs + +#ifdef CL_VERSION_2_0 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueSVMFree)( + cl_command_queue /* command_queue */, cl_uint /* num_svm_pointers */, + void ** /* svm_pointers */, + void(CL_CALLBACK *pfn_free_func)(cl_command_queue /* queue */, + cl_uint /* num_svm_pointers */, + void ** /* svm_pointers[] */, + void * /* user_data */), + void * /* user_data */, cl_uint /* num_events_in_wait_list */, + const cl_event * /* event_wait_list */, + cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueSVMMemcpy)( + cl_command_queue /* command_queue */, cl_bool /* blocking_copy */, + void * /* dst_ptr */, const void * /* src_ptr */, size_t /* size */, + cl_uint /* num_events_in_wait_list */, + const cl_event * /* event_wait_list */, + cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueSVMMemFill)( + cl_command_queue /* command_queue */, void * /* svm_ptr */, + const void * /* pattern */, size_t /* pattern_size */, size_t /* size */, + cl_uint /* num_events_in_wait_list */, + const cl_event * /* event_wait_list */, + cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueSVMMap)( + cl_command_queue /* command_queue */, cl_bool /* blocking_map */, + cl_map_flags /* map_flags */, void * /* svm_ptr */, size_t /* size */, + cl_uint /* num_events_in_wait_list */, + const cl_event * /* event_wait_list */, + cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueSVMUnmap)( + cl_command_queue /* command_queue */, void * /* svm_ptr */, + cl_uint /* num_events_in_wait_list */, + const cl_event * /* event_wait_list */, + cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; + +#else + +typedef void *cl_api_clEnqueueSVMFree; +typedef void *cl_api_clEnqueueSVMMemcpy; +typedef void *cl_api_clEnqueueSVMMemFill; +typedef void *cl_api_clEnqueueSVMMap; +typedef void *cl_api_clEnqueueSVMUnmap; + +#endif + +// Deprecated APIs +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clSetCommandQueueProperty)( + cl_command_queue command_queue, cl_command_queue_properties properties, + cl_bool enable, cl_command_queue_properties *old_properties) + CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateImage2D)( + cl_context context, cl_mem_flags flags, const cl_image_format *image_format, + size_t image_width, size_t image_height, size_t image_row_pitch, + void *host_ptr, cl_int *errcode_ret) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateImage3D)( + cl_context context, cl_mem_flags flags, const cl_image_format *image_format, + size_t image_width, size_t image_height, size_t image_depth, + size_t image_row_pitch, size_t image_slice_pitch, void *host_ptr, + cl_int *errcode_ret) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clUnloadCompiler)(void) + CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueMarker)( + cl_command_queue command_queue, + cl_event *event) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueWaitForEvents)( + cl_command_queue command_queue, cl_uint num_events, + const cl_event *event_list) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueBarrier)( + cl_command_queue command_queue) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + +typedef CL_API_ENTRY void *(CL_API_CALL *cl_api_clGetExtensionFunctionAddress)( + const char *function_name)CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; + +// GL and other APIs +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromGLBuffer)( + cl_context context, cl_mem_flags flags, cl_GLuint bufobj, + int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromGLTexture)( + cl_context context, cl_mem_flags flags, cl_GLenum target, cl_GLint miplevel, + cl_GLuint texture, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromGLTexture2D)( + cl_context context, cl_mem_flags flags, cl_GLenum target, cl_GLint miplevel, + cl_GLuint texture, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromGLTexture3D)( + cl_context context, cl_mem_flags flags, cl_GLenum target, cl_GLint miplevel, + cl_GLuint texture, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromGLRenderbuffer)( + cl_context context, cl_mem_flags flags, cl_GLuint renderbuffer, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetGLObjectInfo)( + cl_mem memobj, cl_gl_object_type *gl_object_type, + cl_GLuint *gl_object_name) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetGLTextureInfo)( + cl_mem memobj, cl_gl_texture_info param_name, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueAcquireGLObjects)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueReleaseGLObjects)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +/* cl_khr_gl_sharing */ +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetGLContextInfoKHR)( + const cl_context_properties *properties, cl_gl_context_info param_name, + size_t param_value_size, void *param_value, size_t *param_value_size_ret); + +/* cl_khr_gl_event */ +typedef CL_API_ENTRY cl_event(CL_API_CALL *cl_api_clCreateEventFromGLsyncKHR)( + cl_context context, cl_GLsync sync, cl_int *errcode_ret); + +#if defined(_WIN32) + +/* cl_khr_d3d10_sharing */ + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetDeviceIDsFromD3D10KHR)( + cl_platform_id platform, cl_d3d10_device_source_khr d3d_device_source, + void *d3d_object, cl_d3d10_device_set_khr d3d_device_set, + cl_uint num_entries, cl_device_id *devices, + cl_uint *num_devices) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromD3D10BufferKHR)( + cl_context context, cl_mem_flags flags, ID3D10Buffer *resource, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromD3D10Texture2DKHR)( + cl_context context, cl_mem_flags flags, ID3D10Texture2D *resource, + UINT subresource, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromD3D10Texture3DKHR)( + cl_context context, cl_mem_flags flags, ID3D10Texture3D *resource, + UINT subresource, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clEnqueueAcquireD3D10ObjectsKHR)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clEnqueueReleaseD3D10ObjectsKHR)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_0; + +extern CL_API_ENTRY cl_int CL_API_CALL clGetDeviceIDsFromD3D10KHR( + cl_platform_id platform, cl_d3d10_device_source_khr d3d_device_source, + void *d3d_object, cl_d3d10_device_set_khr d3d_device_set, + cl_uint num_entries, cl_device_id *devices, cl_uint *num_devices); + +extern CL_API_ENTRY cl_mem CL_API_CALL +clCreateFromD3D10BufferKHR(cl_context context, cl_mem_flags flags, + ID3D10Buffer *resource, cl_int *errcode_ret); + +extern CL_API_ENTRY cl_mem CL_API_CALL clCreateFromD3D10Texture2DKHR( + cl_context context, cl_mem_flags flags, ID3D10Texture2D *resource, + UINT subresource, cl_int *errcode_ret); + +extern CL_API_ENTRY cl_mem CL_API_CALL clCreateFromD3D10Texture3DKHR( + cl_context context, cl_mem_flags flags, ID3D10Texture3D *resource, + UINT subresource, cl_int *errcode_ret); + +extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueAcquireD3D10ObjectsKHR( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); + +extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueReleaseD3D10ObjectsKHR( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); + +/* cl_khr_d3d11_sharing */ +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetDeviceIDsFromD3D11KHR)( + cl_platform_id platform, cl_d3d11_device_source_khr d3d_device_source, + void *d3d_object, cl_d3d11_device_set_khr d3d_device_set, + cl_uint num_entries, cl_device_id *devices, + cl_uint *num_devices) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromD3D11BufferKHR)( + cl_context context, cl_mem_flags flags, ID3D11Buffer *resource, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromD3D11Texture2DKHR)( + cl_context context, cl_mem_flags flags, ID3D11Texture2D *resource, + UINT subresource, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromD3D11Texture3DKHR)( + cl_context context, cl_mem_flags flags, ID3D11Texture3D *resource, + UINT subresource, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clEnqueueAcquireD3D11ObjectsKHR)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clEnqueueReleaseD3D11ObjectsKHR)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +/* cl_khr_dx9_media_sharing */ +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clGetDeviceIDsFromDX9MediaAdapterKHR)( + cl_platform_id platform, cl_uint num_media_adapters, + cl_dx9_media_adapter_type_khr *media_adapters_type, void *media_adapters, + cl_dx9_media_adapter_set_khr media_adapter_set, cl_uint num_entries, + cl_device_id *devices, cl_uint *num_devices) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromDX9MediaSurfaceKHR)( + cl_context context, cl_mem_flags flags, + cl_dx9_media_adapter_type_khr adapter_type, void *surface_info, + cl_uint plane, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clEnqueueAcquireDX9MediaSurfacesKHR)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clEnqueueReleaseDX9MediaSurfacesKHR)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_1_2; + +/* cl_khr_d3d11_sharing */ +extern CL_API_ENTRY cl_int CL_API_CALL clGetDeviceIDsFromD3D11KHR( + cl_platform_id platform, cl_d3d11_device_source_khr d3d_device_source, + void *d3d_object, cl_d3d11_device_set_khr d3d_device_set, + cl_uint num_entries, cl_device_id *devices, cl_uint *num_devices); + +extern CL_API_ENTRY cl_mem CL_API_CALL +clCreateFromD3D11BufferKHR(cl_context context, cl_mem_flags flags, + ID3D11Buffer *resource, cl_int *errcode_ret); + +extern CL_API_ENTRY cl_mem CL_API_CALL clCreateFromD3D11Texture2DKHR( + cl_context context, cl_mem_flags flags, ID3D11Texture2D *resource, + UINT subresource, cl_int *errcode_ret); + +extern CL_API_ENTRY cl_mem CL_API_CALL clCreateFromD3D11Texture3DKHR( + cl_context context, cl_mem_flags flags, ID3D11Texture3D *resource, + UINT subresource, cl_int *errcode_ret); + +extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueAcquireD3D11ObjectsKHR( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); + +extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueReleaseD3D11ObjectsKHR( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); + +/* cl_khr_dx9_media_sharing */ +extern CL_API_ENTRY cl_int CL_API_CALL clGetDeviceIDsFromDX9MediaAdapterKHR( + cl_platform_id platform, cl_uint num_media_adapters, + cl_dx9_media_adapter_type_khr *media_adapter_type, void *media_adapters, + cl_dx9_media_adapter_set_khr media_adapter_set, cl_uint num_entries, + cl_device_id *devices, cl_uint *num_devices); + +extern CL_API_ENTRY cl_mem CL_API_CALL clCreateFromDX9MediaSurfaceKHR( + cl_context context, cl_mem_flags flags, + cl_dx9_media_adapter_type_khr adapter_type, void *surface_info, + cl_uint plane, cl_int *errcode_ret); + +extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueAcquireDX9MediaSurfacesKHR( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); + +extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueReleaseDX9MediaSurfacesKHR( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); + +#else + +/* cl_khr_d3d10_sharing */ +typedef void *cl_api_clGetDeviceIDsFromD3D10KHR; +typedef void *cl_api_clCreateFromD3D10BufferKHR; +typedef void *cl_api_clCreateFromD3D10Texture2DKHR; +typedef void *cl_api_clCreateFromD3D10Texture3DKHR; +typedef void *cl_api_clEnqueueAcquireD3D10ObjectsKHR; +typedef void *cl_api_clEnqueueReleaseD3D10ObjectsKHR; + +/* cl_khr_d3d11_sharing */ +typedef void *cl_api_clGetDeviceIDsFromD3D11KHR; +typedef void *cl_api_clCreateFromD3D11BufferKHR; +typedef void *cl_api_clCreateFromD3D11Texture2DKHR; +typedef void *cl_api_clCreateFromD3D11Texture3DKHR; +typedef void *cl_api_clEnqueueAcquireD3D11ObjectsKHR; +typedef void *cl_api_clEnqueueReleaseD3D11ObjectsKHR; + +/* cl_khr_dx9_media_sharing */ +typedef void *cl_api_clCreateFromDX9MediaSurfaceKHR; +typedef void *cl_api_clEnqueueAcquireDX9MediaSurfacesKHR; +typedef void *cl_api_clEnqueueReleaseDX9MediaSurfacesKHR; +typedef void *cl_api_clGetDeviceIDsFromDX9MediaAdapterKHR; + +#endif + +/* OpenCL 1.1 */ + +#ifdef CL_VERSION_1_1 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clSetEventCallback)( + cl_event /* event */, cl_int /* command_exec_callback_type */, + void(CL_CALLBACK * /* pfn_notify */)(cl_event, cl_int, void *), + void * /* user_data */) CL_API_SUFFIX__VERSION_1_1; + +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateSubBuffer)( + cl_mem /* buffer */, cl_mem_flags /* flags */, + cl_buffer_create_type /* buffer_create_type */, + const void * /* buffer_create_info */, + cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_1; + +typedef CL_API_ENTRY +cl_int(CL_API_CALL *cl_api_clSetMemObjectDestructorCallback)( + cl_mem /* memobj */, + void(CL_CALLBACK * /*pfn_notify*/)(cl_mem /* memobj */, + void * /*user_data*/), + void * /*user_data */) CL_API_SUFFIX__VERSION_1_1; + +typedef CL_API_ENTRY cl_event(CL_API_CALL *cl_api_clCreateUserEvent)( + cl_context /* context */, + cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_1; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clSetUserEventStatus)( + cl_event /* event */, + cl_int /* execution_status */) CL_API_SUFFIX__VERSION_1_1; + +#else + +typedef void *cl_api_clSetEventCallback; +typedef void *cl_api_clCreateSubBuffer; +typedef void *cl_api_clSetMemObjectDestructorCallback; +typedef void *cl_api_clCreateUserEvent; +typedef void *cl_api_clSetUserEventStatus; + +#endif + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clCreateSubDevicesEXT)( + cl_device_id in_device, + const cl_device_partition_property_ext *partition_properties, + cl_uint num_entries, cl_device_id *out_devices, cl_uint *num_devices); + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clRetainDeviceEXT)( + cl_device_id device) CL_API_SUFFIX__VERSION_1_0; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clReleaseDeviceEXT)( + cl_device_id device) CL_API_SUFFIX__VERSION_1_0; + +/* cl_khr_egl_image */ +typedef CL_API_ENTRY cl_mem(CL_API_CALL *cl_api_clCreateFromEGLImageKHR)( + cl_context context, CLeglDisplayKHR display, CLeglImageKHR image, + cl_mem_flags flags, const cl_egl_image_properties_khr *properties, + cl_int *errcode_ret); + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueAcquireEGLObjectsKHR)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueReleaseEGLObjectsKHR)( + cl_command_queue command_queue, cl_uint num_objects, + const cl_mem *mem_objects, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, cl_event *event); + +/* cl_khr_egl_event */ +typedef CL_API_ENTRY cl_event(CL_API_CALL *cl_api_clCreateEventFromEGLSyncKHR)( + cl_context context, CLeglSyncKHR sync, CLeglDisplayKHR display, + cl_int *errcode_ret); + +#ifdef CL_VERSION_2_1 + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clSetDefaultDeviceCommandQueue)( + cl_context context, cl_device_id device, + cl_command_queue command_queue) CL_API_SUFFIX__VERSION_2_1; + +typedef CL_API_ENTRY cl_program(CL_API_CALL *cl_api_clCreateProgramWithIL)( + cl_context context, const void *il, size_t length, + cl_int *errcode_ret) CL_API_SUFFIX__VERSION_2_1; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetKernelSubGroupInfo)( + cl_kernel kernel, cl_device_id device, cl_kernel_sub_group_info param_name, + size_t input_value_size, const void *input_value, size_t param_value_size, + void *param_value, size_t *param_value_size_ret) CL_API_SUFFIX__VERSION_2_1; + +typedef CL_API_ENTRY cl_kernel(CL_API_CALL *cl_api_clCloneKernel)( + cl_kernel source_kernel, cl_int *errcode_ret) CL_API_SUFFIX__VERSION_2_1; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clEnqueueSVMMigrateMem)( + cl_command_queue command_queue, cl_uint num_svm_pointers, + const void **svm_pointers, const size_t *sizes, + cl_mem_migration_flags flags, cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) CL_API_SUFFIX__VERSION_2_1; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetDeviceAndHostTimer)( + cl_device_id device, cl_ulong *device_timestamp, + cl_ulong *host_timestamp) CL_API_SUFFIX__VERSION_2_1; + +typedef CL_API_ENTRY cl_int(CL_API_CALL *cl_api_clGetHostTimer)( + cl_device_id device, cl_ulong *host_timestamp) CL_API_SUFFIX__VERSION_2_1; + +#else + +typedef void *cl_api_clSetDefaultDeviceCommandQueue; +typedef void *cl_api_clCreateProgramWithIL; +typedef void *cl_api_clGetKernelSubGroupInfo; +typedef void *cl_api_clCloneKernel; +typedef void *cl_api_clEnqueueSVMMigrateMem; +typedef void *cl_api_clGetDeviceAndHostTimer; +typedef void *cl_api_clGetHostTimer; + +#endif + +/* Vendor dispatch table struture */ + +typedef struct _cl_icd_dispatch { + /* OpenCL 1.0 */ + cl_api_clGetPlatformIDs clGetPlatformIDs; + cl_api_clGetPlatformInfo clGetPlatformInfo; + cl_api_clGetDeviceIDs clGetDeviceIDs; + cl_api_clGetDeviceInfo clGetDeviceInfo; + cl_api_clCreateContext clCreateContext; + cl_api_clCreateContextFromType clCreateContextFromType; + cl_api_clRetainContext clRetainContext; + cl_api_clReleaseContext clReleaseContext; + cl_api_clGetContextInfo clGetContextInfo; + cl_api_clCreateCommandQueue clCreateCommandQueue; + cl_api_clRetainCommandQueue clRetainCommandQueue; + cl_api_clReleaseCommandQueue clReleaseCommandQueue; + cl_api_clGetCommandQueueInfo clGetCommandQueueInfo; + cl_api_clSetCommandQueueProperty clSetCommandQueueProperty; + cl_api_clCreateBuffer clCreateBuffer; + cl_api_clCreateImage2D clCreateImage2D; + cl_api_clCreateImage3D clCreateImage3D; + cl_api_clRetainMemObject clRetainMemObject; + cl_api_clReleaseMemObject clReleaseMemObject; + cl_api_clGetSupportedImageFormats clGetSupportedImageFormats; + cl_api_clGetMemObjectInfo clGetMemObjectInfo; + cl_api_clGetImageInfo clGetImageInfo; + cl_api_clCreateSampler clCreateSampler; + cl_api_clRetainSampler clRetainSampler; + cl_api_clReleaseSampler clReleaseSampler; + cl_api_clGetSamplerInfo clGetSamplerInfo; + cl_api_clCreateProgramWithSource clCreateProgramWithSource; + cl_api_clCreateProgramWithBinary clCreateProgramWithBinary; + cl_api_clRetainProgram clRetainProgram; + cl_api_clReleaseProgram clReleaseProgram; + cl_api_clBuildProgram clBuildProgram; + cl_api_clUnloadCompiler clUnloadCompiler; + cl_api_clGetProgramInfo clGetProgramInfo; + cl_api_clGetProgramBuildInfo clGetProgramBuildInfo; + cl_api_clCreateKernel clCreateKernel; + cl_api_clCreateKernelsInProgram clCreateKernelsInProgram; + cl_api_clRetainKernel clRetainKernel; + cl_api_clReleaseKernel clReleaseKernel; + cl_api_clSetKernelArg clSetKernelArg; + cl_api_clGetKernelInfo clGetKernelInfo; + cl_api_clGetKernelWorkGroupInfo clGetKernelWorkGroupInfo; + cl_api_clWaitForEvents clWaitForEvents; + cl_api_clGetEventInfo clGetEventInfo; + cl_api_clRetainEvent clRetainEvent; + cl_api_clReleaseEvent clReleaseEvent; + cl_api_clGetEventProfilingInfo clGetEventProfilingInfo; + cl_api_clFlush clFlush; + cl_api_clFinish clFinish; + cl_api_clEnqueueReadBuffer clEnqueueReadBuffer; + cl_api_clEnqueueWriteBuffer clEnqueueWriteBuffer; + cl_api_clEnqueueCopyBuffer clEnqueueCopyBuffer; + cl_api_clEnqueueReadImage clEnqueueReadImage; + cl_api_clEnqueueWriteImage clEnqueueWriteImage; + cl_api_clEnqueueCopyImage clEnqueueCopyImage; + cl_api_clEnqueueCopyImageToBuffer clEnqueueCopyImageToBuffer; + cl_api_clEnqueueCopyBufferToImage clEnqueueCopyBufferToImage; + cl_api_clEnqueueMapBuffer clEnqueueMapBuffer; + cl_api_clEnqueueMapImage clEnqueueMapImage; + cl_api_clEnqueueUnmapMemObject clEnqueueUnmapMemObject; + cl_api_clEnqueueNDRangeKernel clEnqueueNDRangeKernel; + cl_api_clEnqueueTask clEnqueueTask; + cl_api_clEnqueueNativeKernel clEnqueueNativeKernel; + cl_api_clEnqueueMarker clEnqueueMarker; + cl_api_clEnqueueWaitForEvents clEnqueueWaitForEvents; + cl_api_clEnqueueBarrier clEnqueueBarrier; + cl_api_clGetExtensionFunctionAddress clGetExtensionFunctionAddress; + cl_api_clCreateFromGLBuffer clCreateFromGLBuffer; + cl_api_clCreateFromGLTexture2D clCreateFromGLTexture2D; + cl_api_clCreateFromGLTexture3D clCreateFromGLTexture3D; + cl_api_clCreateFromGLRenderbuffer clCreateFromGLRenderbuffer; + cl_api_clGetGLObjectInfo clGetGLObjectInfo; + cl_api_clGetGLTextureInfo clGetGLTextureInfo; + cl_api_clEnqueueAcquireGLObjects clEnqueueAcquireGLObjects; + cl_api_clEnqueueReleaseGLObjects clEnqueueReleaseGLObjects; + cl_api_clGetGLContextInfoKHR clGetGLContextInfoKHR; + + /* cl_khr_d3d10_sharing */ + cl_api_clGetDeviceIDsFromD3D10KHR clGetDeviceIDsFromD3D10KHR; + cl_api_clCreateFromD3D10BufferKHR clCreateFromD3D10BufferKHR; + cl_api_clCreateFromD3D10Texture2DKHR clCreateFromD3D10Texture2DKHR; + cl_api_clCreateFromD3D10Texture3DKHR clCreateFromD3D10Texture3DKHR; + cl_api_clEnqueueAcquireD3D10ObjectsKHR clEnqueueAcquireD3D10ObjectsKHR; + cl_api_clEnqueueReleaseD3D10ObjectsKHR clEnqueueReleaseD3D10ObjectsKHR; + + /* OpenCL 1.1 */ + cl_api_clSetEventCallback clSetEventCallback; + cl_api_clCreateSubBuffer clCreateSubBuffer; + cl_api_clSetMemObjectDestructorCallback clSetMemObjectDestructorCallback; + cl_api_clCreateUserEvent clCreateUserEvent; + cl_api_clSetUserEventStatus clSetUserEventStatus; + cl_api_clEnqueueReadBufferRect clEnqueueReadBufferRect; + cl_api_clEnqueueWriteBufferRect clEnqueueWriteBufferRect; + cl_api_clEnqueueCopyBufferRect clEnqueueCopyBufferRect; + + /* cl_ext_device_fission */ + cl_api_clCreateSubDevicesEXT clCreateSubDevicesEXT; + cl_api_clRetainDeviceEXT clRetainDeviceEXT; + cl_api_clReleaseDeviceEXT clReleaseDeviceEXT; + + /* cl_khr_gl_event */ + cl_api_clCreateEventFromGLsyncKHR clCreateEventFromGLsyncKHR; + + /* OpenCL 1.2 */ + cl_api_clCreateSubDevices clCreateSubDevices; + cl_api_clRetainDevice clRetainDevice; + cl_api_clReleaseDevice clReleaseDevice; + cl_api_clCreateImage clCreateImage; + cl_api_clCreateProgramWithBuiltInKernels clCreateProgramWithBuiltInKernels; + cl_api_clCompileProgram clCompileProgram; + cl_api_clLinkProgram clLinkProgram; + cl_api_clUnloadPlatformCompiler clUnloadPlatformCompiler; + cl_api_clGetKernelArgInfo clGetKernelArgInfo; + cl_api_clEnqueueFillBuffer clEnqueueFillBuffer; + cl_api_clEnqueueFillImage clEnqueueFillImage; + cl_api_clEnqueueMigrateMemObjects clEnqueueMigrateMemObjects; + cl_api_clEnqueueMarkerWithWaitList clEnqueueMarkerWithWaitList; + cl_api_clEnqueueBarrierWithWaitList clEnqueueBarrierWithWaitList; + cl_api_clGetExtensionFunctionAddressForPlatform + clGetExtensionFunctionAddressForPlatform; + cl_api_clCreateFromGLTexture clCreateFromGLTexture; + + /* cl_khr_d3d11_sharing */ + cl_api_clGetDeviceIDsFromD3D11KHR clGetDeviceIDsFromD3D11KHR; + cl_api_clCreateFromD3D11BufferKHR clCreateFromD3D11BufferKHR; + cl_api_clCreateFromD3D11Texture2DKHR clCreateFromD3D11Texture2DKHR; + cl_api_clCreateFromD3D11Texture3DKHR clCreateFromD3D11Texture3DKHR; + cl_api_clCreateFromDX9MediaSurfaceKHR clCreateFromDX9MediaSurfaceKHR; + cl_api_clEnqueueAcquireD3D11ObjectsKHR clEnqueueAcquireD3D11ObjectsKHR; + cl_api_clEnqueueReleaseD3D11ObjectsKHR clEnqueueReleaseD3D11ObjectsKHR; + + /* cl_khr_dx9_media_sharing */ + cl_api_clGetDeviceIDsFromDX9MediaAdapterKHR + clGetDeviceIDsFromDX9MediaAdapterKHR; + cl_api_clEnqueueAcquireDX9MediaSurfacesKHR + clEnqueueAcquireDX9MediaSurfacesKHR; + cl_api_clEnqueueReleaseDX9MediaSurfacesKHR + clEnqueueReleaseDX9MediaSurfacesKHR; + + /* cl_khr_egl_image */ + cl_api_clCreateFromEGLImageKHR clCreateFromEGLImageKHR; + cl_api_clEnqueueAcquireEGLObjectsKHR clEnqueueAcquireEGLObjectsKHR; + cl_api_clEnqueueReleaseEGLObjectsKHR clEnqueueReleaseEGLObjectsKHR; + + /* cl_khr_egl_event */ + cl_api_clCreateEventFromEGLSyncKHR clCreateEventFromEGLSyncKHR; + + /* OpenCL 2.0 */ + cl_api_clCreateCommandQueueWithProperties clCreateCommandQueueWithProperties; + cl_api_clCreatePipe clCreatePipe; + cl_api_clGetPipeInfo clGetPipeInfo; + cl_api_clSVMAlloc clSVMAlloc; + cl_api_clSVMFree clSVMFree; + cl_api_clEnqueueSVMFree clEnqueueSVMFree; + cl_api_clEnqueueSVMMemcpy clEnqueueSVMMemcpy; + cl_api_clEnqueueSVMMemFill clEnqueueSVMMemFill; + cl_api_clEnqueueSVMMap clEnqueueSVMMap; + cl_api_clEnqueueSVMUnmap clEnqueueSVMUnmap; + cl_api_clCreateSamplerWithProperties clCreateSamplerWithProperties; + cl_api_clSetKernelArgSVMPointer clSetKernelArgSVMPointer; + cl_api_clSetKernelExecInfo clSetKernelExecInfo; + + /* cl_khr_sub_groups */ + cl_api_clGetKernelSubGroupInfoKHR clGetKernelSubGroupInfoKHR; + + /* OpenCL 2.1 */ + cl_api_clCloneKernel clCloneKernel; + cl_api_clCreateProgramWithIL clCreateProgramWithIL; + cl_api_clEnqueueSVMMigrateMem clEnqueueSVMMigrateMem; + cl_api_clGetDeviceAndHostTimer clGetDeviceAndHostTimer; + cl_api_clGetHostTimer clGetHostTimer; + cl_api_clGetKernelSubGroupInfo clGetKernelSubGroupInfo; + cl_api_clSetDefaultDeviceCommandQueue clSetDefaultDeviceCommandQueue; + + /* OpenCL 2.2 */ + cl_api_clSetProgramReleaseCallback clSetProgramReleaseCallback; + cl_api_clSetProgramSpecializationConstant clSetProgramSpecializationConstant; +} cl_icd_dispatch; + +#ifdef __cplusplus +} +#endif + +#endif /* #ifndef OPENCL_CL_ICD_H */ diff --git a/amdocl/CL/cl_platform.h b/amdocl/CL/cl_platform.h index 15702d2f72..7f4ddea5b3 100644 --- a/amdocl/CL/cl_platform.h +++ b/amdocl/CL/cl_platform.h @@ -1,5 +1,5 @@ /********************************************************************************** - * Copyright (c) 2008-2015 The Khronos Group Inc. + * Copyright (c) 2008-2018 The Khronos Group Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and/or associated documentation files (the @@ -26,15 +26,10 @@ * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. **********************************************************************************/ -/* $Revision: 11803 $ on $Date: 2010-06-25 10:02:12 -0700 (Fri, 25 Jun 2010) $ */ - #ifndef __CL_PLATFORM_H #define __CL_PLATFORM_H -#ifdef __APPLE__ - /* Contains #defines for AVAILABLE_MAC_OS_X_VERSION_10_6_AND_LATER below */ - #include -#endif +#include #ifdef __cplusplus extern "C" { @@ -58,139 +53,70 @@ extern "C" { * deprecation but is deprecated in versions later than 1.1. */ -#ifdef __APPLE__ - #define CL_EXTENSION_WEAK_LINK __attribute__((weak_import)) - #define CL_API_SUFFIX__VERSION_1_0 AVAILABLE_MAC_OS_X_VERSION_10_6_AND_LATER - #define CL_EXT_SUFFIX__VERSION_1_0 CL_EXTENSION_WEAK_LINK AVAILABLE_MAC_OS_X_VERSION_10_6_AND_LATER - #define CL_API_SUFFIX__VERSION_1_1 AVAILABLE_MAC_OS_X_VERSION_10_7_AND_LATER - #define GCL_API_SUFFIX__VERSION_1_1 AVAILABLE_MAC_OS_X_VERSION_10_7_AND_LATER - #define CL_EXT_SUFFIX__VERSION_1_1 CL_EXTENSION_WEAK_LINK AVAILABLE_MAC_OS_X_VERSION_10_7_AND_LATER - #define CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED CL_EXTENSION_WEAK_LINK AVAILABLE_MAC_OS_X_VERSION_10_6_AND_LATER_BUT_DEPRECATED_IN_MAC_OS_X_VERSION_10_7 - - #ifdef AVAILABLE_MAC_OS_X_VERSION_10_8_AND_LATER - #define CL_API_SUFFIX__VERSION_1_2 AVAILABLE_MAC_OS_X_VERSION_10_8_AND_LATER - #define GCL_API_SUFFIX__VERSION_1_2 AVAILABLE_MAC_OS_X_VERSION_10_8_AND_LATER - #define CL_EXT_SUFFIX__VERSION_1_2 CL_EXTENSION_WEAK_LINK AVAILABLE_MAC_OS_X_VERSION_10_8_AND_LATER - #define CL_EXT_PREFIX__VERSION_1_1_DEPRECATED - #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED CL_EXTENSION_WEAK_LINK AVAILABLE_MAC_OS_X_VERSION_10_7_AND_LATER_BUT_DEPRECATED_IN_MAC_OS_X_VERSION_10_8 - #else - #warning This path should never happen outside of internal operating system development. AvailabilityMacros do not function correctly here! - #define CL_API_SUFFIX__VERSION_1_2 AVAILABLE_MAC_OS_X_VERSION_10_7_AND_LATER - #define GCL_API_SUFFIX__VERSION_1_2 AVAILABLE_MAC_OS_X_VERSION_10_7_AND_LATER - #define CL_EXT_SUFFIX__VERSION_1_2 CL_EXTENSION_WEAK_LINK AVAILABLE_MAC_OS_X_VERSION_10_7_AND_LATER - #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED CL_EXTENSION_WEAK_LINK AVAILABLE_MAC_OS_X_VERSION_10_7_AND_LATER - #endif +#define CL_EXTENSION_WEAK_LINK +#define CL_API_SUFFIX__VERSION_1_0 +#define CL_EXT_SUFFIX__VERSION_1_0 +#define CL_API_SUFFIX__VERSION_1_1 +#define CL_EXT_SUFFIX__VERSION_1_1 +#define CL_API_SUFFIX__VERSION_1_2 +#define CL_EXT_SUFFIX__VERSION_1_2 +#define CL_API_SUFFIX__VERSION_2_0 +#define CL_EXT_SUFFIX__VERSION_2_0 +#define CL_API_SUFFIX__VERSION_2_1 +#define CL_EXT_SUFFIX__VERSION_2_1 +#define CL_API_SUFFIX__VERSION_2_2 +#define CL_EXT_SUFFIX__VERSION_2_2 + + +#ifdef __GNUC__ + #define CL_EXT_SUFFIX_DEPRECATED __attribute__((deprecated)) + #define CL_EXT_PREFIX_DEPRECATED +#elif defined(_WIN32) + #define CL_EXT_SUFFIX_DEPRECATED + #define CL_EXT_PREFIX_DEPRECATED __declspec(deprecated) #else - #define CL_EXTENSION_WEAK_LINK - #define CL_API_SUFFIX__VERSION_1_0 - #define CL_EXT_SUFFIX__VERSION_1_0 - #define CL_API_SUFFIX__VERSION_1_1 - #define CL_EXT_SUFFIX__VERSION_1_1 - #define CL_API_SUFFIX__VERSION_1_2 - #define CL_EXT_SUFFIX__VERSION_1_2 - #define CL_API_SUFFIX__VERSION_2_0 - #define CL_EXT_SUFFIX__VERSION_2_0 - #define CL_API_SUFFIX__VERSION_2_1 - #define CL_EXT_SUFFIX__VERSION_2_1 - #define CL_API_SUFFIX__VERSION_2_2 - #define CL_EXT_SUFFIX__VERSION_2_2 - - #ifdef __GNUC__ - #ifdef CL_USE_DEPRECATED_OPENCL_1_0_APIS - #define CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_0_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED __attribute__((deprecated)) - #define CL_EXT_PREFIX__VERSION_1_0_DEPRECATED - #endif - - #ifdef CL_USE_DEPRECATED_OPENCL_1_1_APIS - #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_1_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED __attribute__((deprecated)) - #define CL_EXT_PREFIX__VERSION_1_1_DEPRECATED - #endif + #define CL_EXT_SUFFIX_DEPRECATED + #define CL_EXT_PREFIX_DEPRECATED +#endif - #ifdef CL_USE_DEPRECATED_OPENCL_1_2_APIS - #define CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_2_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED __attribute__((deprecated)) - #define CL_EXT_PREFIX__VERSION_1_2_DEPRECATED - #endif +#ifdef CL_USE_DEPRECATED_OPENCL_1_0_APIS + #define CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED + #define CL_EXT_PREFIX__VERSION_1_0_DEPRECATED +#else + #define CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED CL_EXT_SUFFIX_DEPRECATED + #define CL_EXT_PREFIX__VERSION_1_0_DEPRECATED CL_EXT_PREFIX_DEPRECATED +#endif - #ifdef CL_USE_DEPRECATED_OPENCL_2_0_APIS - #define CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED - #define CL_EXT_PREFIX__VERSION_2_0_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED __attribute__((deprecated)) - #define CL_EXT_PREFIX__VERSION_2_0_DEPRECATED - #endif +#ifdef CL_USE_DEPRECATED_OPENCL_1_1_APIS + #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED + #define CL_EXT_PREFIX__VERSION_1_1_DEPRECATED +#else + #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED CL_EXT_SUFFIX_DEPRECATED + #define CL_EXT_PREFIX__VERSION_1_1_DEPRECATED CL_EXT_PREFIX_DEPRECATED +#endif - #ifdef CL_USE_DEPRECATED_OPENCL_2_1_APIS - #define CL_EXT_SUFFIX__VERSION_2_1_DEPRECATED - #define CL_EXT_PREFIX__VERSION_2_1_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_2_1_DEPRECATED __attribute__((deprecated)) - #define CL_EXT_PREFIX__VERSION_2_1_DEPRECATED - #endif - #elif defined(_WIN32) - #ifdef CL_USE_DEPRECATED_OPENCL_1_0_APIS - #define CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_0_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_0_DEPRECATED __declspec(deprecated) - #endif - - #ifdef CL_USE_DEPRECATED_OPENCL_1_1_APIS - #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_1_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_1_DEPRECATED __declspec(deprecated) - #endif - - #ifdef CL_USE_DEPRECATED_OPENCL_1_2_APIS - #define CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_2_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_2_DEPRECATED __declspec(deprecated) - #endif +#ifdef CL_USE_DEPRECATED_OPENCL_1_2_APIS + #define CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED + #define CL_EXT_PREFIX__VERSION_1_2_DEPRECATED +#else + #define CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED CL_EXT_SUFFIX_DEPRECATED + #define CL_EXT_PREFIX__VERSION_1_2_DEPRECATED CL_EXT_PREFIX_DEPRECATED + #endif - #ifdef CL_USE_DEPRECATED_OPENCL_2_0_APIS - #define CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED - #define CL_EXT_PREFIX__VERSION_2_0_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED - #define CL_EXT_PREFIX__VERSION_2_0_DEPRECATED __declspec(deprecated) - #endif +#ifdef CL_USE_DEPRECATED_OPENCL_2_0_APIS + #define CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED + #define CL_EXT_PREFIX__VERSION_2_0_DEPRECATED +#else + #define CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED CL_EXT_SUFFIX_DEPRECATED + #define CL_EXT_PREFIX__VERSION_2_0_DEPRECATED CL_EXT_PREFIX_DEPRECATED +#endif - #ifdef CL_USE_DEPRECATED_OPENCL_2_1_APIS - #define CL_EXT_SUFFIX__VERSION_2_1_DEPRECATED - #define CL_EXT_PREFIX__VERSION_2_1_DEPRECATED - #else - #define CL_EXT_SUFFIX__VERSION_2_1_DEPRECATED - #define CL_EXT_PREFIX__VERSION_2_1_DEPRECATED __declspec(deprecated) - #endif - #else - #define CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_0_DEPRECATED - - #define CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_1_DEPRECATED - - #define CL_EXT_SUFFIX__VERSION_1_2_DEPRECATED - #define CL_EXT_PREFIX__VERSION_1_2_DEPRECATED - - #define CL_EXT_SUFFIX__VERSION_2_0_DEPRECATED - #define CL_EXT_PREFIX__VERSION_2_0_DEPRECATED - - #define CL_EXT_SUFFIX__VERSION_2_1_DEPRECATED - #define CL_EXT_PREFIX__VERSION_2_1_DEPRECATED - #endif +#ifdef CL_USE_DEPRECATED_OPENCL_2_1_APIS + #define CL_EXT_SUFFIX__VERSION_2_1_DEPRECATED + #define CL_EXT_PREFIX__VERSION_2_1_DEPRECATED +#else + #define CL_EXT_SUFFIX__VERSION_2_1_DEPRECATED CL_EXT_SUFFIX_DEPRECATED + #define CL_EXT_PREFIX__VERSION_2_1_DEPRECATED CL_EXT_PREFIX_DEPRECATED #endif #if (defined (_WIN32) && defined(_MSC_VER)) @@ -300,16 +226,16 @@ typedef double cl_double; /* scalar types */ typedef int8_t cl_char; typedef uint8_t cl_uchar; -typedef int16_t cl_short __attribute__((aligned(2))); -typedef uint16_t cl_ushort __attribute__((aligned(2))); -typedef int32_t cl_int __attribute__((aligned(4))); -typedef uint32_t cl_uint __attribute__((aligned(4))); -typedef int64_t cl_long __attribute__((aligned(8))); -typedef uint64_t cl_ulong __attribute__((aligned(8))); +typedef int16_t cl_short; +typedef uint16_t cl_ushort; +typedef int32_t cl_int; +typedef uint32_t cl_uint; +typedef int64_t cl_long; +typedef uint64_t cl_ulong; -typedef uint16_t cl_half __attribute__((aligned(2))); -typedef float cl_float __attribute__((aligned(4))); -typedef double cl_double __attribute__((aligned(8))); +typedef uint16_t cl_half; +typedef float cl_float; +typedef double cl_double; /* Macro names and corresponding values defined by OpenCL */ #define CL_CHAR_BIT 8 @@ -397,7 +323,7 @@ typedef double cl_double __attribute__((aligned(8))); #define CL_HUGE_VALF ((cl_float) 1e50) #define CL_HUGE_VAL ((cl_double) 1e500) float nanf( const char * ); - #define CL_NAN nanf( "" ) + #define CL_NAN nanf( "" ) #endif #define CL_MAXFLOAT CL_FLT_MAX #define CL_INFINITY CL_HUGE_VALF @@ -412,17 +338,17 @@ typedef int cl_GLint; typedef unsigned int cl_GLenum; /* - * Vector types + * Vector types * - * Note: OpenCL requires that all types be naturally aligned. + * Note: OpenCL requires that all types be naturally aligned. * This means that vector types must be naturally aligned. * For example, a vector of four floats must be aligned to - * a 16 byte boundary (calculated as 4 * the natural 4-byte + * a 16 byte boundary (calculated as 4 * the natural 4-byte * alignment of the float). The alignment qualifiers here * will only function properly if your compiler supports them * and if you don't actively work to defeat them. For example, * in order for a cl_float4 to be 16 byte aligned in a struct, - * the start of the struct must itself be 16-byte aligned. + * the start of the struct must itself be 16-byte aligned. * * Maintaining proper alignment is the user's responsibility. */ @@ -430,13 +356,13 @@ typedef unsigned int cl_GLenum; /* Define basic vector types */ #if defined( __VEC__ ) #include /* may be omitted depending on compiler. AltiVec spec provides no way to detect whether the header is required. */ - typedef vector unsigned char __cl_uchar16; - typedef vector signed char __cl_char16; - typedef vector unsigned short __cl_ushort8; - typedef vector signed short __cl_short8; - typedef vector unsigned int __cl_uint4; - typedef vector signed int __cl_int4; - typedef vector float __cl_float4; + typedef __vector unsigned char __cl_uchar16; + typedef __vector signed char __cl_char16; + typedef __vector unsigned short __cl_ushort8; + typedef __vector signed short __cl_short8; + typedef __vector unsigned int __cl_uint4; + typedef __vector signed int __cl_int4; + typedef __vector float __cl_float4; #define __CL_UCHAR16__ 1 #define __CL_CHAR16__ 1 #define __CL_USHORT8__ 1 @@ -536,7 +462,7 @@ typedef unsigned int cl_GLenum; #if defined( __MINGW64__ ) #include #else - #include + #include #endif #if defined( __GNUC__ ) typedef cl_float __cl_float8 __attribute__((vector_size(32))); @@ -605,7 +531,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_char s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_char lo, hi; }; #endif -#if defined( __CL_CHAR2__) +#if defined( __CL_CHAR2__) __cl_char2 v2; #endif }cl_char2; @@ -618,10 +544,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_char s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_char2 lo, hi; }; #endif -#if defined( __CL_CHAR2__) +#if defined( __CL_CHAR2__) __cl_char2 v2[2]; #endif -#if defined( __CL_CHAR4__) +#if defined( __CL_CHAR4__) __cl_char4 v4; #endif }cl_char4; @@ -637,10 +563,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_char s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_char4 lo, hi; }; #endif -#if defined( __CL_CHAR2__) +#if defined( __CL_CHAR2__) __cl_char2 v2[4]; #endif -#if defined( __CL_CHAR4__) +#if defined( __CL_CHAR4__) __cl_char4 v4[2]; #endif #if defined( __CL_CHAR8__ ) @@ -656,10 +582,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_char s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_char8 lo, hi; }; #endif -#if defined( __CL_CHAR2__) +#if defined( __CL_CHAR2__) __cl_char2 v2[8]; #endif -#if defined( __CL_CHAR4__) +#if defined( __CL_CHAR4__) __cl_char4 v4[4]; #endif #if defined( __CL_CHAR8__ ) @@ -680,7 +606,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_uchar s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_uchar lo, hi; }; #endif -#if defined( __cl_uchar2__) +#if defined( __cl_uchar2__) __cl_uchar2 v2; #endif }cl_uchar2; @@ -693,10 +619,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_uchar s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_uchar2 lo, hi; }; #endif -#if defined( __CL_UCHAR2__) +#if defined( __CL_UCHAR2__) __cl_uchar2 v2[2]; #endif -#if defined( __CL_UCHAR4__) +#if defined( __CL_UCHAR4__) __cl_uchar4 v4; #endif }cl_uchar4; @@ -712,10 +638,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_uchar s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_uchar4 lo, hi; }; #endif -#if defined( __CL_UCHAR2__) +#if defined( __CL_UCHAR2__) __cl_uchar2 v2[4]; #endif -#if defined( __CL_UCHAR4__) +#if defined( __CL_UCHAR4__) __cl_uchar4 v4[2]; #endif #if defined( __CL_UCHAR8__ ) @@ -731,10 +657,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_uchar s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_uchar8 lo, hi; }; #endif -#if defined( __CL_UCHAR2__) +#if defined( __CL_UCHAR2__) __cl_uchar2 v2[8]; #endif -#if defined( __CL_UCHAR4__) +#if defined( __CL_UCHAR4__) __cl_uchar4 v4[4]; #endif #if defined( __CL_UCHAR8__ ) @@ -755,7 +681,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_short s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_short lo, hi; }; #endif -#if defined( __CL_SHORT2__) +#if defined( __CL_SHORT2__) __cl_short2 v2; #endif }cl_short2; @@ -768,10 +694,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_short s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_short2 lo, hi; }; #endif -#if defined( __CL_SHORT2__) +#if defined( __CL_SHORT2__) __cl_short2 v2[2]; #endif -#if defined( __CL_SHORT4__) +#if defined( __CL_SHORT4__) __cl_short4 v4; #endif }cl_short4; @@ -787,10 +713,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_short s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_short4 lo, hi; }; #endif -#if defined( __CL_SHORT2__) +#if defined( __CL_SHORT2__) __cl_short2 v2[4]; #endif -#if defined( __CL_SHORT4__) +#if defined( __CL_SHORT4__) __cl_short4 v4[2]; #endif #if defined( __CL_SHORT8__ ) @@ -806,10 +732,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_short s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_short8 lo, hi; }; #endif -#if defined( __CL_SHORT2__) +#if defined( __CL_SHORT2__) __cl_short2 v2[8]; #endif -#if defined( __CL_SHORT4__) +#if defined( __CL_SHORT4__) __cl_short4 v4[4]; #endif #if defined( __CL_SHORT8__ ) @@ -830,7 +756,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_ushort s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_ushort lo, hi; }; #endif -#if defined( __CL_USHORT2__) +#if defined( __CL_USHORT2__) __cl_ushort2 v2; #endif }cl_ushort2; @@ -843,10 +769,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_ushort s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_ushort2 lo, hi; }; #endif -#if defined( __CL_USHORT2__) +#if defined( __CL_USHORT2__) __cl_ushort2 v2[2]; #endif -#if defined( __CL_USHORT4__) +#if defined( __CL_USHORT4__) __cl_ushort4 v4; #endif }cl_ushort4; @@ -862,10 +788,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_ushort s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_ushort4 lo, hi; }; #endif -#if defined( __CL_USHORT2__) +#if defined( __CL_USHORT2__) __cl_ushort2 v2[4]; #endif -#if defined( __CL_USHORT4__) +#if defined( __CL_USHORT4__) __cl_ushort4 v4[2]; #endif #if defined( __CL_USHORT8__ ) @@ -881,10 +807,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_ushort s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_ushort8 lo, hi; }; #endif -#if defined( __CL_USHORT2__) +#if defined( __CL_USHORT2__) __cl_ushort2 v2[8]; #endif -#if defined( __CL_USHORT4__) +#if defined( __CL_USHORT4__) __cl_ushort4 v4[4]; #endif #if defined( __CL_USHORT8__ ) @@ -905,7 +831,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_half s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_half lo, hi; }; #endif -#if defined( __CL_HALF2__) +#if defined( __CL_HALF2__) __cl_half2 v2; #endif }cl_half2; @@ -918,10 +844,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_half s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_half2 lo, hi; }; #endif -#if defined( __CL_HALF2__) +#if defined( __CL_HALF2__) __cl_half2 v2[2]; #endif -#if defined( __CL_HALF4__) +#if defined( __CL_HALF4__) __cl_half4 v4; #endif }cl_half4; @@ -937,10 +863,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_half s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_half4 lo, hi; }; #endif -#if defined( __CL_HALF2__) +#if defined( __CL_HALF2__) __cl_half2 v2[4]; #endif -#if defined( __CL_HALF4__) +#if defined( __CL_HALF4__) __cl_half4 v4[2]; #endif #if defined( __CL_HALF8__ ) @@ -956,10 +882,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_half s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_half8 lo, hi; }; #endif -#if defined( __CL_HALF2__) +#if defined( __CL_HALF2__) __cl_half2 v2[8]; #endif -#if defined( __CL_HALF4__) +#if defined( __CL_HALF4__) __cl_half4 v4[4]; #endif #if defined( __CL_HALF8__ ) @@ -979,7 +905,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_int s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_int lo, hi; }; #endif -#if defined( __CL_INT2__) +#if defined( __CL_INT2__) __cl_int2 v2; #endif }cl_int2; @@ -992,10 +918,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_int s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_int2 lo, hi; }; #endif -#if defined( __CL_INT2__) +#if defined( __CL_INT2__) __cl_int2 v2[2]; #endif -#if defined( __CL_INT4__) +#if defined( __CL_INT4__) __cl_int4 v4; #endif }cl_int4; @@ -1011,10 +937,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_int s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_int4 lo, hi; }; #endif -#if defined( __CL_INT2__) +#if defined( __CL_INT2__) __cl_int2 v2[4]; #endif -#if defined( __CL_INT4__) +#if defined( __CL_INT4__) __cl_int4 v4[2]; #endif #if defined( __CL_INT8__ ) @@ -1030,10 +956,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_int s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_int8 lo, hi; }; #endif -#if defined( __CL_INT2__) +#if defined( __CL_INT2__) __cl_int2 v2[8]; #endif -#if defined( __CL_INT4__) +#if defined( __CL_INT4__) __cl_int4 v4[4]; #endif #if defined( __CL_INT8__ ) @@ -1054,7 +980,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_uint s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_uint lo, hi; }; #endif -#if defined( __CL_UINT2__) +#if defined( __CL_UINT2__) __cl_uint2 v2; #endif }cl_uint2; @@ -1067,10 +993,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_uint s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_uint2 lo, hi; }; #endif -#if defined( __CL_UINT2__) +#if defined( __CL_UINT2__) __cl_uint2 v2[2]; #endif -#if defined( __CL_UINT4__) +#if defined( __CL_UINT4__) __cl_uint4 v4; #endif }cl_uint4; @@ -1086,10 +1012,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_uint s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_uint4 lo, hi; }; #endif -#if defined( __CL_UINT2__) +#if defined( __CL_UINT2__) __cl_uint2 v2[4]; #endif -#if defined( __CL_UINT4__) +#if defined( __CL_UINT4__) __cl_uint4 v4[2]; #endif #if defined( __CL_UINT8__ ) @@ -1105,10 +1031,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_uint s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_uint8 lo, hi; }; #endif -#if defined( __CL_UINT2__) +#if defined( __CL_UINT2__) __cl_uint2 v2[8]; #endif -#if defined( __CL_UINT4__) +#if defined( __CL_UINT4__) __cl_uint4 v4[4]; #endif #if defined( __CL_UINT8__ ) @@ -1128,7 +1054,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_long s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_long lo, hi; }; #endif -#if defined( __CL_LONG2__) +#if defined( __CL_LONG2__) __cl_long2 v2; #endif }cl_long2; @@ -1141,10 +1067,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_long s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_long2 lo, hi; }; #endif -#if defined( __CL_LONG2__) +#if defined( __CL_LONG2__) __cl_long2 v2[2]; #endif -#if defined( __CL_LONG4__) +#if defined( __CL_LONG4__) __cl_long4 v4; #endif }cl_long4; @@ -1160,10 +1086,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_long s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_long4 lo, hi; }; #endif -#if defined( __CL_LONG2__) +#if defined( __CL_LONG2__) __cl_long2 v2[4]; #endif -#if defined( __CL_LONG4__) +#if defined( __CL_LONG4__) __cl_long4 v4[2]; #endif #if defined( __CL_LONG8__ ) @@ -1179,10 +1105,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_long s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_long8 lo, hi; }; #endif -#if defined( __CL_LONG2__) +#if defined( __CL_LONG2__) __cl_long2 v2[8]; #endif -#if defined( __CL_LONG4__) +#if defined( __CL_LONG4__) __cl_long4 v4[4]; #endif #if defined( __CL_LONG8__ ) @@ -1203,7 +1129,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_ulong s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_ulong lo, hi; }; #endif -#if defined( __CL_ULONG2__) +#if defined( __CL_ULONG2__) __cl_ulong2 v2; #endif }cl_ulong2; @@ -1216,10 +1142,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_ulong s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_ulong2 lo, hi; }; #endif -#if defined( __CL_ULONG2__) +#if defined( __CL_ULONG2__) __cl_ulong2 v2[2]; #endif -#if defined( __CL_ULONG4__) +#if defined( __CL_ULONG4__) __cl_ulong4 v4; #endif }cl_ulong4; @@ -1235,10 +1161,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_ulong s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_ulong4 lo, hi; }; #endif -#if defined( __CL_ULONG2__) +#if defined( __CL_ULONG2__) __cl_ulong2 v2[4]; #endif -#if defined( __CL_ULONG4__) +#if defined( __CL_ULONG4__) __cl_ulong4 v4[2]; #endif #if defined( __CL_ULONG8__ ) @@ -1254,10 +1180,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_ulong s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_ulong8 lo, hi; }; #endif -#if defined( __CL_ULONG2__) +#if defined( __CL_ULONG2__) __cl_ulong2 v2[8]; #endif -#if defined( __CL_ULONG4__) +#if defined( __CL_ULONG4__) __cl_ulong4 v4[4]; #endif #if defined( __CL_ULONG8__ ) @@ -1279,7 +1205,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_float s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_float lo, hi; }; #endif -#if defined( __CL_FLOAT2__) +#if defined( __CL_FLOAT2__) __cl_float2 v2; #endif }cl_float2; @@ -1292,10 +1218,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_float s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_float2 lo, hi; }; #endif -#if defined( __CL_FLOAT2__) +#if defined( __CL_FLOAT2__) __cl_float2 v2[2]; #endif -#if defined( __CL_FLOAT4__) +#if defined( __CL_FLOAT4__) __cl_float4 v4; #endif }cl_float4; @@ -1311,10 +1237,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_float s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_float4 lo, hi; }; #endif -#if defined( __CL_FLOAT2__) +#if defined( __CL_FLOAT2__) __cl_float2 v2[4]; #endif -#if defined( __CL_FLOAT4__) +#if defined( __CL_FLOAT4__) __cl_float4 v4[2]; #endif #if defined( __CL_FLOAT8__ ) @@ -1330,10 +1256,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_float s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_float8 lo, hi; }; #endif -#if defined( __CL_FLOAT2__) +#if defined( __CL_FLOAT2__) __cl_float2 v2[8]; #endif -#if defined( __CL_FLOAT4__) +#if defined( __CL_FLOAT4__) __cl_float4 v4[4]; #endif #if defined( __CL_FLOAT8__ ) @@ -1354,7 +1280,7 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_double s0, s1; }; __CL_ANON_STRUCT__ struct{ cl_double lo, hi; }; #endif -#if defined( __CL_DOUBLE2__) +#if defined( __CL_DOUBLE2__) __cl_double2 v2; #endif }cl_double2; @@ -1367,10 +1293,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_double s0, s1, s2, s3; }; __CL_ANON_STRUCT__ struct{ cl_double2 lo, hi; }; #endif -#if defined( __CL_DOUBLE2__) +#if defined( __CL_DOUBLE2__) __cl_double2 v2[2]; #endif -#if defined( __CL_DOUBLE4__) +#if defined( __CL_DOUBLE4__) __cl_double4 v4; #endif }cl_double4; @@ -1386,10 +1312,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_double s0, s1, s2, s3, s4, s5, s6, s7; }; __CL_ANON_STRUCT__ struct{ cl_double4 lo, hi; }; #endif -#if defined( __CL_DOUBLE2__) +#if defined( __CL_DOUBLE2__) __cl_double2 v2[4]; #endif -#if defined( __CL_DOUBLE4__) +#if defined( __CL_DOUBLE4__) __cl_double4 v4[2]; #endif #if defined( __CL_DOUBLE8__ ) @@ -1405,10 +1331,10 @@ typedef union __CL_ANON_STRUCT__ struct{ cl_double s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, sA, sB, sC, sD, sE, sF; }; __CL_ANON_STRUCT__ struct{ cl_double8 lo, hi; }; #endif -#if defined( __CL_DOUBLE2__) +#if defined( __CL_DOUBLE2__) __cl_double2 v2[8]; #endif -#if defined( __CL_DOUBLE4__) +#if defined( __CL_DOUBLE4__) __cl_double4 v4[4]; #endif #if defined( __CL_DOUBLE8__ ) @@ -1419,9 +1345,9 @@ typedef union #endif }cl_double16; -/* Macro to facilitate debugging +/* Macro to facilitate debugging * Usage: - * Place CL_PROGRAM_STRING_DEBUG_INFO on the line before the first line of your source. + * Place CL_PROGRAM_STRING_DEBUG_INFO on the line before the first line of your source. * The first line ends with: CL_PROGRAM_STRING_DEBUG_INFO \" * Each line thereafter of OpenCL C source must end with: \n\ * The last line ends in "; @@ -1436,13 +1362,13 @@ typedef union * } \n\ * "; * - * This should correctly set up the line, (column) and file information for your source + * This should correctly set up the line, (column) and file information for your source * string so you can do source level debugging. */ #define __CL_STRINGIFY( _x ) # _x #define _CL_STRINGIFY( _x ) __CL_STRINGIFY( _x ) -#define CL_PROGRAM_STRING_DEBUG_INFO "#line " _CL_STRINGIFY(__LINE__) " \"" __FILE__ "\" \n\n" - +#define CL_PROGRAM_STRING_DEBUG_INFO "#line " _CL_STRINGIFY(__LINE__) " \"" __FILE__ "\" \n\n" + #ifdef __cplusplus } #endif diff --git a/amdocl/CL/cl_version.h b/amdocl/CL/cl_version.h new file mode 100644 index 0000000000..bb766cb9bb --- /dev/null +++ b/amdocl/CL/cl_version.h @@ -0,0 +1,86 @@ +/******************************************************************************* + * Copyright (c) 2018 The Khronos Group Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and/or associated documentation files (the + * "Materials"), to deal in the Materials without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Materials, and to + * permit persons to whom the Materials are furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Materials. + * + * MODIFICATIONS TO THIS FILE MAY MEAN IT NO LONGER ACCURATELY REFLECTS + * KHRONOS STANDARDS. THE UNMODIFIED, NORMATIVE VERSIONS OF KHRONOS + * SPECIFICATIONS AND HEADER INFORMATION ARE LOCATED AT + * https://www.khronos.org/registry/ + * + * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. + ******************************************************************************/ + +#ifndef __CL_VERSION_H +#define __CL_VERSION_H + +/* Detect which version to target */ +#if !defined(CL_TARGET_OPENCL_VERSION) +#pragma message("cl_version.h: CL_TARGET_OPENCL_VERSION is not defined. Defaulting to 220 (OpenCL 2.2)") +#define CL_TARGET_OPENCL_VERSION 220 +#endif +#if CL_TARGET_OPENCL_VERSION != 100 && \ + CL_TARGET_OPENCL_VERSION != 110 && \ + CL_TARGET_OPENCL_VERSION != 120 && \ + CL_TARGET_OPENCL_VERSION != 200 && \ + CL_TARGET_OPENCL_VERSION != 210 && \ + CL_TARGET_OPENCL_VERSION != 220 +#pragma message("cl_version: CL_TARGET_OPENCL_VERSION is not a valid value (100, 110, 120, 200, 210, 220). Defaulting to 220 (OpenCL 2.2)") +#undef CL_TARGET_OPENCL_VERSION +#define CL_TARGET_OPENCL_VERSION 220 +#endif + + +/* OpenCL Version */ +#if CL_TARGET_OPENCL_VERSION >= 220 && !defined(CL_VERSION_2_2) +#define CL_VERSION_2_2 1 +#endif +#if CL_TARGET_OPENCL_VERSION >= 210 && !defined(CL_VERSION_2_1) +#define CL_VERSION_2_1 1 +#endif +#if CL_TARGET_OPENCL_VERSION >= 200 && !defined(CL_VERSION_2_0) +#define CL_VERSION_2_0 1 +#endif +#if CL_TARGET_OPENCL_VERSION >= 120 && !defined(CL_VERSION_1_2) +#define CL_VERSION_1_2 1 +#endif +#if CL_TARGET_OPENCL_VERSION >= 110 && !defined(CL_VERSION_1_1) +#define CL_VERSION_1_1 1 +#endif +#if CL_TARGET_OPENCL_VERSION >= 100 && !defined(CL_VERSION_1_0) +#define CL_VERSION_1_0 1 +#endif + +/* Allow deprecated APIs for older OpenCL versions. */ +#if CL_TARGET_OPENCL_VERSION <= 210 && !defined(CL_USE_DEPRECATED_OPENCL_2_1_APIS) +#define CL_USE_DEPRECATED_OPENCL_2_1_APIS +#endif +#if CL_TARGET_OPENCL_VERSION <= 200 && !defined(CL_USE_DEPRECATED_OPENCL_2_0_APIS) +#define CL_USE_DEPRECATED_OPENCL_2_0_APIS +#endif +#if CL_TARGET_OPENCL_VERSION <= 120 && !defined(CL_USE_DEPRECATED_OPENCL_1_2_APIS) +#define CL_USE_DEPRECATED_OPENCL_1_2_APIS +#endif +#if CL_TARGET_OPENCL_VERSION <= 110 && !defined(CL_USE_DEPRECATED_OPENCL_1_1_APIS) +#define CL_USE_DEPRECATED_OPENCL_1_1_APIS +#endif +#if CL_TARGET_OPENCL_VERSION <= 100 && !defined(CL_USE_DEPRECATED_OPENCL_1_0_APIS) +#define CL_USE_DEPRECATED_OPENCL_1_0_APIS +#endif + +#endif /* __CL_VERSION_H */ diff --git a/amdocl/CL/opencl.h b/amdocl/CL/opencl.h index 9855cd75e7..143d1d2dc6 100644 --- a/amdocl/CL/opencl.h +++ b/amdocl/CL/opencl.h @@ -35,25 +35,13 @@ extern "C" { #endif -#ifdef __APPLE__ - -#include -#include -#include -#include - -#else - #include #include #include #include -#endif - #ifdef __cplusplus } #endif #endif /* __OPENCL_H */ - diff --git a/amdocl/cl_icd.cpp b/amdocl/cl_icd.cpp new file mode 100644 index 0000000000..ec2cb48d7d --- /dev/null +++ b/amdocl/cl_icd.cpp @@ -0,0 +1,293 @@ +/* Copyright (c) 2008-present Advanced Micro Devices, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. */ + +#include "cl_common.hpp" +#include "vdi_common.hpp" +#ifdef _WIN32 +#include +#include "cl_d3d9_amd.hpp" +#include "cl_d3d10_amd.hpp" +#include "cl_d3d11_amd.hpp" +#endif //_WIN32 + +#include + +#include + +amd::PlatformIDS amd::PlatformID::Platform = //{ NULL }; + {amd::ICDDispatchedObject::icdVendorDispatch_}; + +static cl_int CL_API_CALL icdGetPlatformInfo(cl_platform_id platform, cl_platform_info param_name, + size_t param_value_size, void* param_value, + size_t* param_value_size_ret) { + return clGetPlatformInfo(NULL, param_name, param_value_size, param_value, param_value_size_ret); +} + +static cl_int CL_API_CALL icdGetDeviceIDs(cl_platform_id platform, cl_device_type device_type, + cl_uint num_entries, cl_device_id* devices, + cl_uint* num_devices) { + return clGetDeviceIDs(NULL, device_type, num_entries, devices, num_devices); +} + +static cl_int CL_API_CALL icdGetDeviceInfo(cl_device_id device, cl_device_info param_name, + size_t param_value_size, void* param_value, + size_t* param_value_size_ret) { + if (param_name == CL_DEVICE_PLATFORM) { + // Return the ICD platform instead of the default NULL platform. + cl_platform_id platform = reinterpret_cast(&amd::PlatformID::Platform); + return amd::clGetInfo(platform, param_value_size, param_value, param_value_size_ret); + } + + return clGetDeviceInfo(device, param_name, param_value_size, param_value, param_value_size_ret); +} + +cl_icd_dispatch amd::ICDDispatchedObject::icdVendorDispatch_[] = { + {NULL /* should not get called */, icdGetPlatformInfo, icdGetDeviceIDs, icdGetDeviceInfo, + clCreateContext, clCreateContextFromType, clRetainContext, clReleaseContext, clGetContextInfo, + clCreateCommandQueue, clRetainCommandQueue, clReleaseCommandQueue, clGetCommandQueueInfo, + clSetCommandQueueProperty, clCreateBuffer, clCreateImage2D, clCreateImage3D, clRetainMemObject, + clReleaseMemObject, clGetSupportedImageFormats, clGetMemObjectInfo, clGetImageInfo, + clCreateSampler, clRetainSampler, clReleaseSampler, clGetSamplerInfo, + clCreateProgramWithSource, clCreateProgramWithBinary, clRetainProgram, clReleaseProgram, + clBuildProgram, clUnloadCompiler, clGetProgramInfo, clGetProgramBuildInfo, clCreateKernel, + clCreateKernelsInProgram, clRetainKernel, clReleaseKernel, clSetKernelArg, clGetKernelInfo, + clGetKernelWorkGroupInfo, clWaitForEvents, clGetEventInfo, clRetainEvent, clReleaseEvent, + clGetEventProfilingInfo, clFlush, clFinish, clEnqueueReadBuffer, clEnqueueWriteBuffer, + clEnqueueCopyBuffer, clEnqueueReadImage, clEnqueueWriteImage, clEnqueueCopyImage, + clEnqueueCopyImageToBuffer, clEnqueueCopyBufferToImage, clEnqueueMapBuffer, clEnqueueMapImage, + clEnqueueUnmapMemObject, clEnqueueNDRangeKernel, clEnqueueTask, clEnqueueNativeKernel, + clEnqueueMarker, clEnqueueWaitForEvents, clEnqueueBarrier, clGetExtensionFunctionAddress, + clCreateFromGLBuffer, clCreateFromGLTexture2D, clCreateFromGLTexture3D, + clCreateFromGLRenderbuffer, clGetGLObjectInfo, clGetGLTextureInfo, clEnqueueAcquireGLObjects, + clEnqueueReleaseGLObjects, clGetGLContextInfoKHR, + WINDOWS_SWITCH(clGetDeviceIDsFromD3D10KHR, NULL), + WINDOWS_SWITCH(clCreateFromD3D10BufferKHR, NULL), + WINDOWS_SWITCH(clCreateFromD3D10Texture2DKHR, NULL), + WINDOWS_SWITCH(clCreateFromD3D10Texture3DKHR, NULL), + WINDOWS_SWITCH(clEnqueueAcquireD3D10ObjectsKHR, NULL), + WINDOWS_SWITCH(clEnqueueReleaseD3D10ObjectsKHR, NULL), clSetEventCallback, clCreateSubBuffer, + clSetMemObjectDestructorCallback, clCreateUserEvent, clSetUserEventStatus, + clEnqueueReadBufferRect, clEnqueueWriteBufferRect, clEnqueueCopyBufferRect, + NULL, NULL, NULL, clCreateEventFromGLsyncKHR, + + /* OpenCL 1.2*/ + clCreateSubDevices, clRetainDevice, clReleaseDevice, clCreateImage, + clCreateProgramWithBuiltInKernels, clCompileProgram, clLinkProgram, clUnloadPlatformCompiler, + clGetKernelArgInfo, clEnqueueFillBuffer, clEnqueueFillImage, clEnqueueMigrateMemObjects, + clEnqueueMarkerWithWaitList, clEnqueueBarrierWithWaitList, + clGetExtensionFunctionAddressForPlatform, clCreateFromGLTexture, + + WINDOWS_SWITCH(clGetDeviceIDsFromD3D11KHR, NULL), + WINDOWS_SWITCH(clCreateFromD3D11BufferKHR, NULL), + WINDOWS_SWITCH(clCreateFromD3D11Texture2DKHR, NULL), + WINDOWS_SWITCH(clCreateFromD3D11Texture3DKHR, NULL), + WINDOWS_SWITCH(clCreateFromDX9MediaSurfaceKHR, NULL), + WINDOWS_SWITCH(clEnqueueAcquireD3D11ObjectsKHR, NULL), + WINDOWS_SWITCH(clEnqueueReleaseD3D11ObjectsKHR, NULL), + + WINDOWS_SWITCH(clGetDeviceIDsFromDX9MediaAdapterKHR, + NULL), // KHRpfn_clGetDeviceIDsFromDX9MediaAdapterKHR + // clGetDeviceIDsFromDX9MediaAdapterKHR; + WINDOWS_SWITCH( + clEnqueueAcquireDX9MediaSurfacesKHR, + NULL), // KHRpfn_clEnqueueAcquireDX9MediaSurfacesKHR clEnqueueAcquireDX9MediaSurfacesKHR; + WINDOWS_SWITCH( + clEnqueueReleaseDX9MediaSurfacesKHR, + NULL), // KHRpfn_clEnqueueReleaseDX9MediaSurfacesKHR clEnqueueReleaseDX9MediaSurfacesKHR; + + NULL, + NULL, NULL, NULL, + + clCreateCommandQueueWithProperties, clCreatePipe, clGetPipeInfo, clSVMAlloc, clSVMFree, + clEnqueueSVMFree, clEnqueueSVMMemcpy, clEnqueueSVMMemFill, clEnqueueSVMMap, clEnqueueSVMUnmap, + clCreateSamplerWithProperties, clSetKernelArgSVMPointer, clSetKernelExecInfo, + clGetKernelSubGroupInfo, + clCloneKernel, + clCreateProgramWithIL, + clEnqueueSVMMigrateMem, + clGetDeviceAndHostTimer, + clGetHostTimer, + clGetKernelSubGroupInfo, + clSetDefaultDeviceCommandQueue, + + clSetProgramReleaseCallback, + clSetProgramSpecializationConstant }}; + +#if defined(ATI_OS_WIN) +#include + +#pragma comment(lib, "shlwapi.lib") + +static bool ShouldLoadPlatform() { + // Get the OpenCL ICD registry values + HKEY platformsKey = NULL; + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "SOFTWARE\\Khronos\\OpenCL\\Vendors", 0, KEY_READ, + &platformsKey) != ERROR_SUCCESS) + return true; + + std::vector registryValues; + DWORD dwIndex = 0; + while (true) { + char cszLibraryName[1024] = {0}; + DWORD dwLibraryNameSize = sizeof(cszLibraryName); + DWORD dwLibraryNameType = 0; + DWORD dwValue = 0; + DWORD dwValueSize = sizeof(dwValue); + + if (RegEnumValueA(platformsKey, dwIndex++, cszLibraryName, &dwLibraryNameSize, NULL, + &dwLibraryNameType, (LPBYTE)&dwValue, &dwValueSize) != ERROR_SUCCESS) + break; + // Require that the value be a DWORD and equal zero + if (dwLibraryNameType != REG_DWORD || dwValue != 0) { + continue; + } + registryValues.push_back(cszLibraryName); + } + RegCloseKey(platformsKey); + + HMODULE hm = NULL; + if (!GetModuleHandleExA( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCSTR)&ShouldLoadPlatform, &hm)) + return true; + + char cszDllPath[1024] = {0}; + if (!GetModuleFileNameA(hm, cszDllPath, sizeof(cszDllPath))) return true; + + // If we are loaded from the DriverStore, then there should be a registry + // value matching our current module absolute path. + if (std::find(registryValues.begin(), registryValues.end(), cszDllPath) == registryValues.end()) + return true; + + LPSTR cszFileName; + char buffer[1024] = {0}; + if (!GetFullPathNameA(cszDllPath, sizeof(buffer), buffer, &cszFileName)) return true; + + // We found an absolute path in the registry that matched this DLL, now + // check if there is also an entry with the same filename. + if (std::find(registryValues.begin(), registryValues.end(), cszFileName) == registryValues.end()) + return true; + + // Lastly, check if there is a DLL with the same name in the System folder. + char cszSystemPath[1024] = {0}; +#if defined(ATI_BITS_32) + if (!GetSystemWow64DirectoryA(cszSystemPath, sizeof(cszSystemPath))) +#endif // defined(ATI_BITS_32) + if (!GetSystemDirectoryA(cszSystemPath, sizeof(cszSystemPath))) return true; + + std::string systemDllPath; + systemDllPath.append(cszSystemPath).append("\\").append(cszFileName); + if (!PathFileExistsA(systemDllPath.c_str())) { + return true; + } + + // If we get here, then all 3 conditions are true: + // - An entry in the registry with an absolute path matches the current DLL + // - An entry in the registry with a relative path matches the current DLL + // - A DLL with the same name was found in the system directory + // + // We should not load this platform! + + return false; +} + +#else + +#include + +// If there is only one platform, load it. +// If there is more than one platform, only load platforms that have visible devices +// If all platforms have no devices available, only load the PAL platform +static bool ShouldLoadPlatform() { + bool shouldLoad = true; + + if (!amd::Runtime::initialized()) { + amd::Runtime::init(); + } + const int numDevices = amd::Device::numDevices(CL_DEVICE_TYPE_GPU, false); + + void *otherPlatform = nullptr; + if (amd::IS_LEGACY) { + otherPlatform = dlopen("libamdocl64.so", RTLD_LAZY); + if (otherPlatform != nullptr) { // Present platform exists + shouldLoad = numDevices > 0; + } + } else { + otherPlatform = dlopen("libamdocl-orca64.so", RTLD_LAZY); + if (otherPlatform != nullptr) { // Legacy platform exists + // gcc4.8 doesn't support casting void* to a function pointer + // Work around this by creating a typedef untill we upgrade the compiler + typedef void*(*clGetFunctionAddress_t)(const char *); + typedef cl_int(*clIcdGetPlatformIDs_t)(cl_uint, cl_platform_id *, cl_uint *); + + clGetFunctionAddress_t legacyGetFunctionAddress = + reinterpret_cast(dlsym(otherPlatform, "clGetExtensionFunctionAddress")); + clIcdGetPlatformIDs_t legacyGetPlatformIDs = + reinterpret_cast(legacyGetFunctionAddress("clIcdGetPlatformIDsKHR")); + + cl_uint numLegacyPlatforms = 0; + legacyGetPlatformIDs(0, nullptr, &numLegacyPlatforms); + + shouldLoad = (numDevices > 0) || (numLegacyPlatforms == 0); + } + } + + if (otherPlatform != nullptr) { + dlclose(otherPlatform); + } + + return shouldLoad; +} + +#endif // defined(ATI_OS_WIN) + +CL_API_ENTRY cl_int CL_API_CALL clIcdGetPlatformIDsKHR(cl_uint num_entries, + cl_platform_id* platforms, + cl_uint* num_platforms) { + if (((num_entries > 0 || num_platforms == NULL) && platforms == NULL) || + (num_entries == 0 && platforms != NULL)) { + return CL_INVALID_VALUE; + } + + static bool shouldLoad = true; + + static std::once_flag initOnce; + std::call_once(initOnce, [](){ shouldLoad = ShouldLoadPlatform(); }); + + if (!shouldLoad) { + *not_null(num_platforms) = 0; + return CL_SUCCESS; + } + + if (!amd::Runtime::initialized()) { + amd::Runtime::init(); + } + + if (num_platforms != NULL && platforms == NULL) { + *num_platforms = 1; + return CL_SUCCESS; + } + + assert(platforms != NULL && "check the code above"); + *platforms = reinterpret_cast(&amd::PlatformID::Platform); + + *not_null(num_platforms) = 1; + return CL_SUCCESS; +} diff --git a/amdocl/icd/loader/icd_dispatch.h b/amdocl/icd/loader/icd_dispatch.h index 22af3354c6..84a3e305a7 100644 --- a/amdocl/icd/loader/icd_dispatch.h +++ b/amdocl/icd/loader/icd_dispatch.h @@ -51,1410 +51,7 @@ #include #include #include - -/* - * - * function pointer typedefs - * - */ - -// Platform APIs -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetPlatformIDs)( - cl_uint num_entries, - cl_platform_id * platforms, - cl_uint * num_platforms) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetPlatformInfo)( - cl_platform_id platform, - cl_platform_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -// Device APIs -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetDeviceIDs)( - cl_platform_id platform, - cl_device_type device_type, - cl_uint num_entries, - cl_device_id * devices, - cl_uint * num_devices) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetDeviceInfo)( - cl_device_id device, - cl_device_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clCreateSubDevices)( - cl_device_id in_device, - const cl_device_partition_property * partition_properties, - cl_uint num_entries, - cl_device_id * out_devices, - cl_uint * num_devices); - -typedef CL_API_ENTRY cl_int (CL_API_CALL * KHRpfn_clRetainDevice)( - cl_device_id device) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL * KHRpfn_clReleaseDevice)( - cl_device_id device) CL_API_SUFFIX__VERSION_1_2; - -// Context APIs -typedef CL_API_ENTRY cl_context (CL_API_CALL *KHRpfn_clCreateContext)( - const cl_context_properties * properties, - cl_uint num_devices, - const cl_device_id * devices, - void (CL_CALLBACK *pfn_notify)(const char *, const void *, size_t, void *), - void * user_data, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_context (CL_API_CALL *KHRpfn_clCreateContextFromType)( - const cl_context_properties * properties, - cl_device_type device_type, - void (CL_CALLBACK *pfn_notify)(const char *, const void *, size_t, void *), - void * user_data, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clRetainContext)( - cl_context context) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clReleaseContext)( - cl_context context) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetContextInfo)( - cl_context context, - cl_context_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -// Command Queue APIs -typedef CL_API_ENTRY cl_command_queue (CL_API_CALL *KHRpfn_clCreateCommandQueue)( - cl_context context, - cl_device_id device, - cl_command_queue_properties properties, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_command_queue (CL_API_CALL *KHRpfn_clCreateCommandQueueWithProperties)( - cl_context /* context */, - cl_device_id /* device */, - const cl_queue_properties * /* properties */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clRetainCommandQueue)( - cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clReleaseCommandQueue)( - cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetCommandQueueInfo)( - cl_command_queue command_queue, - cl_command_queue_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -// Memory Object APIs -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateBuffer)( - cl_context context, - cl_mem_flags flags, - size_t size, - void * host_ptr, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateImage)( - cl_context context, - cl_mem_flags flags, - const cl_image_format * image_format, - const cl_image_desc * image_desc, - void * host_ptr, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clRetainMemObject)(cl_mem memobj) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clReleaseMemObject)(cl_mem memobj) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetSupportedImageFormats)( - cl_context context, - cl_mem_flags flags, - cl_mem_object_type image_type, - cl_uint num_entries, - cl_image_format * image_formats, - cl_uint * num_image_formats) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetMemObjectInfo)( - cl_mem memobj, - cl_mem_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetImageInfo)( - cl_mem image, - cl_image_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreatePipe)( - cl_context /* context */, - cl_mem_flags /* flags */, - cl_uint /* pipe_packet_size */, - cl_uint /* pipe_max_packets */, - const cl_pipe_properties * /* properties */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetPipeInfo)( - cl_mem /* pipe */, - cl_pipe_info /* param_name */, - size_t /* param_value_size */, - void * /* param_value */, - size_t * /* param_value_size_ret */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY void * (CL_API_CALL *KHRpfn_clSVMAlloc)( - cl_context /* context */, - cl_svm_mem_flags /* flags */, - size_t /* size */, - unsigned int /* alignment */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY void (CL_API_CALL *KHRpfn_clSVMFree)( - cl_context /* context */, - void * /* svm_pointer */) CL_API_SUFFIX__VERSION_2_0; - -// Sampler APIs -typedef CL_API_ENTRY cl_sampler (CL_API_CALL *KHRpfn_clCreateSampler)( - cl_context context, - cl_bool normalized_coords, - cl_addressing_mode addressing_mode, - cl_filter_mode filter_mode, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clRetainSampler)(cl_sampler sampler) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clReleaseSampler)(cl_sampler sampler) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetSamplerInfo)( - cl_sampler sampler, - cl_sampler_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_sampler (CL_API_CALL *KHRpfn_clCreateSamplerWithProperties)( - cl_context /* context */, - const cl_sampler_properties * /* sampler_properties */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_2_0; - -// Program Object APIs -typedef CL_API_ENTRY cl_program (CL_API_CALL *KHRpfn_clCreateProgramWithSource)( - cl_context context, - cl_uint count, - const char ** strings, - const size_t * lengths, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_program (CL_API_CALL *KHRpfn_clCreateProgramWithBinary)( - cl_context context, - cl_uint num_devices, - const cl_device_id * device_list, - const size_t * lengths, - const unsigned char ** binaries, - cl_int * binary_status, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_program (CL_API_CALL *KHRpfn_clCreateProgramWithBuiltInKernels)( - cl_context context, - cl_uint num_devices, - const cl_device_id * device_list, - const char * kernel_names, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clRetainProgram)(cl_program program) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clReleaseProgram)(cl_program program) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clBuildProgram)( - cl_program program, - cl_uint num_devices, - const cl_device_id * device_list, - const char * options, - void (CL_CALLBACK *pfn_notify)(cl_program program, void * user_data), - void * user_data) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clCompileProgram)( - cl_program program, - cl_uint num_devices, - const cl_device_id * device_list, - const char * options, - cl_uint num_input_headers, - const cl_program * input_headers, - const char ** header_include_names, - void (CL_CALLBACK * pfn_notify)(cl_program program, void * user_data), - void * user_data) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_program (CL_API_CALL *KHRpfn_clLinkProgram)( - cl_context context, - cl_uint num_devices, - const cl_device_id * device_list, - const char * options, - cl_uint num_input_programs, - const cl_program * input_programs, - void (CL_CALLBACK * pfn_notify)(cl_program program, void * user_data), - void * user_data, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetProgramSpecializationConstant)( - cl_program program, - cl_uint spec_id, - size_t spec_size, - const void* spec_value) CL_API_SUFFIX__VERSION_2_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetProgramReleaseCallback)( - cl_program program, - void (CL_CALLBACK * pfn_notify)(cl_program program, void * user_data), - void * user_data) CL_API_SUFFIX__VERSION_2_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clUnloadPlatformCompiler)( - cl_platform_id platform) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetProgramInfo)( - cl_program program, - cl_program_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetProgramBuildInfo)( - cl_program program, - cl_device_id device, - cl_program_build_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -// Kernel Object APIs -typedef CL_API_ENTRY cl_kernel (CL_API_CALL *KHRpfn_clCreateKernel)( - cl_program program, - const char * kernel_name, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clCreateKernelsInProgram)( - cl_program program, - cl_uint num_kernels, - cl_kernel * kernels, - cl_uint * num_kernels_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clRetainKernel)(cl_kernel kernel) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clReleaseKernel)(cl_kernel kernel) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetKernelArg)( - cl_kernel kernel, - cl_uint arg_index, - size_t arg_size, - const void * arg_value) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetKernelInfo)( - cl_kernel kernel, - cl_kernel_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetKernelArgInfo)( - cl_kernel kernel, - cl_uint arg_indx, - cl_kernel_arg_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetKernelWorkGroupInfo)( - cl_kernel kernel, - cl_device_id device, - cl_kernel_work_group_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetKernelArgSVMPointer)( - cl_kernel /* kernel */, - cl_uint /* arg_index */, - const void * /* arg_value */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetKernelExecInfo)( - cl_kernel /* kernel */, - cl_kernel_exec_info /* param_name */, - size_t /* param_value_size */, - const void * /* param_value */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetKernelSubGroupInfoKHR)( - cl_kernel /* in_kernel */, - cl_device_id /*in_device*/, - cl_kernel_sub_group_info /* param_name */, - size_t /*input_value_size*/, - const void * /*input_value*/, - size_t /*param_value_size*/, - void* /*param_value*/, - size_t* /*param_value_size_ret*/) CL_EXT_SUFFIX__VERSION_2_0; - -// Event Object APIs -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clWaitForEvents)( - cl_uint num_events, - const cl_event * event_list) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetEventInfo)( - cl_event event, - cl_event_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clRetainEvent)(cl_event event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clReleaseEvent)(cl_event event) CL_API_SUFFIX__VERSION_1_0; - -// Profiling APIs -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetEventProfilingInfo)( - cl_event event, - cl_profiling_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -// Flush and Finish APIs -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clFlush)(cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clFinish)(cl_command_queue command_queue) CL_API_SUFFIX__VERSION_1_0; - -// Enqueued Commands APIs -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueReadBuffer)( - cl_command_queue command_queue, - cl_mem buffer, - cl_bool blocking_read, - size_t offset, - size_t cb, - void * ptr, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueReadBufferRect)( - cl_command_queue command_queue, - cl_mem buffer, - cl_bool blocking_read, - const size_t * buffer_origin, - const size_t * host_origin, - const size_t * region, - size_t buffer_row_pitch, - size_t buffer_slice_pitch, - size_t host_row_pitch, - size_t host_slice_pitch, - void * ptr, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueWriteBuffer)( - cl_command_queue command_queue, - cl_mem buffer, - cl_bool blocking_write, - size_t offset, - size_t cb, - const void * ptr, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueWriteBufferRect)( - cl_command_queue command_queue, - cl_mem buffer, - cl_bool blocking_read, - const size_t * buffer_origin, - const size_t * host_origin, - const size_t * region, - size_t buffer_row_pitch, - size_t buffer_slice_pitch, - size_t host_row_pitch, - size_t host_slice_pitch, - const void * ptr, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueFillBuffer)( - cl_command_queue command_queue, - cl_mem buffer, - const void * pattern, - size_t pattern_size, - size_t offset, - size_t cb, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueCopyBuffer)( - cl_command_queue command_queue, - cl_mem src_buffer, - cl_mem dst_buffer, - size_t src_offset, - size_t dst_offset, - size_t cb, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueCopyBufferRect)( - cl_command_queue command_queue, - cl_mem src_buffer, - cl_mem dst_buffer, - const size_t * src_origin, - const size_t * dst_origin, - const size_t * region, - size_t src_row_pitch, - size_t src_slice_pitch, - size_t dst_row_pitch, - size_t dst_slice_pitch, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueReadImage)( - cl_command_queue command_queue, - cl_mem image, - cl_bool blocking_read, - const size_t * origin, - const size_t * region, - size_t row_pitch, - size_t slice_pitch, - void * ptr, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueWriteImage)( - cl_command_queue command_queue, - cl_mem image, - cl_bool blocking_write, - const size_t * origin, - const size_t * region, - size_t input_row_pitch, - size_t input_slice_pitch, - const void * ptr, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueFillImage)( - cl_command_queue command_queue, - cl_mem image, - const void * fill_color, - const size_t origin[3], - const size_t region[3], - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueCopyImage)( - cl_command_queue command_queue, - cl_mem src_image, - cl_mem dst_image, - const size_t * src_origin, - const size_t * dst_origin, - const size_t * region, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueCopyImageToBuffer)( - cl_command_queue command_queue, - cl_mem src_image, - cl_mem dst_buffer, - const size_t * src_origin, - const size_t * region, - size_t dst_offset, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueCopyBufferToImage)( - cl_command_queue command_queue, - cl_mem src_buffer, - cl_mem dst_image, - size_t src_offset, - const size_t * dst_origin, - const size_t * region, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY void * (CL_API_CALL *KHRpfn_clEnqueueMapBuffer)( - cl_command_queue command_queue, - cl_mem buffer, - cl_bool blocking_map, - cl_map_flags map_flags, - size_t offset, - size_t cb, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY void * (CL_API_CALL *KHRpfn_clEnqueueMapImage)( - cl_command_queue command_queue, - cl_mem image, - cl_bool blocking_map, - cl_map_flags map_flags, - const size_t * origin, - const size_t * region, - size_t * image_row_pitch, - size_t * image_slice_pitch, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueUnmapMemObject)( - cl_command_queue command_queue, - cl_mem memobj, - void * mapped_ptr, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueMigrateMemObjects)( - cl_command_queue command_queue, - cl_uint num_mem_objects, - const cl_mem * mem_objects, - cl_mem_migration_flags flags, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueNDRangeKernel)( - cl_command_queue command_queue, - cl_kernel kernel, - cl_uint work_dim, - const size_t * global_work_offset, - const size_t * global_work_size, - const size_t * local_work_size, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueTask)( - cl_command_queue command_queue, - cl_kernel kernel, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueNativeKernel)( - cl_command_queue command_queue, - void (CL_CALLBACK * user_func)(void *), - void * args, - size_t cb_args, - cl_uint num_mem_objects, - const cl_mem * mem_list, - const void ** args_mem_loc, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueMarkerWithWaitList)( - cl_command_queue command_queue, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueBarrierWithWaitList)( - cl_command_queue command_queue, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY void * (CL_API_CALL *KHRpfn_clGetExtensionFunctionAddressForPlatform)( - cl_platform_id platform, - const char * function_name) CL_API_SUFFIX__VERSION_1_2; - -// Shared Virtual Memory APIs -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueSVMFree)( - cl_command_queue /* command_queue */, - cl_uint /* num_svm_pointers */, - void ** /* svm_pointers */, - void (CL_CALLBACK *pfn_free_func)( - cl_command_queue /* queue */, - cl_uint /* num_svm_pointers */, - void ** /* svm_pointers[] */, - void * /* user_data */), - void * /* user_data */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueSVMMemcpy)( - cl_command_queue /* command_queue */, - cl_bool /* blocking_copy */, - void * /* dst_ptr */, - const void * /* src_ptr */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueSVMMemFill)( - cl_command_queue /* command_queue */, - void * /* svm_ptr */, - const void * /* pattern */, - size_t /* pattern_size */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueSVMMap)( - cl_command_queue /* command_queue */, - cl_bool /* blocking_map */, - cl_map_flags /* map_flags */, - void * /* svm_ptr */, - size_t /* size */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueSVMUnmap)( - cl_command_queue /* command_queue */, - void * /* svm_ptr */, - cl_uint /* num_events_in_wait_list */, - const cl_event * /* event_wait_list */, - cl_event * /* event */) CL_API_SUFFIX__VERSION_2_0; - -// Deprecated APIs -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetCommandQueueProperty)( - cl_command_queue command_queue, - cl_command_queue_properties properties, - cl_bool enable, - cl_command_queue_properties * old_properties) CL_EXT_SUFFIX__VERSION_1_0_DEPRECATED; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateImage2D)( - cl_context context, - cl_mem_flags flags, - const cl_image_format * image_format, - size_t image_width, - size_t image_height, - size_t image_row_pitch, - void * host_ptr, - cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateImage3D)( - cl_context context, - cl_mem_flags flags, - const cl_image_format * image_format, - size_t image_width, - size_t image_height, - size_t image_depth, - size_t image_row_pitch, - size_t image_slice_pitch, - void * host_ptr, - cl_int * errcode_ret) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clUnloadCompiler)(void) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueMarker)( - cl_command_queue command_queue, - cl_event * event) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueWaitForEvents)( - cl_command_queue command_queue, - cl_uint num_events, - const cl_event * event_list) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueBarrier)(cl_command_queue command_queue) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - -typedef CL_API_ENTRY void * (CL_API_CALL *KHRpfn_clGetExtensionFunctionAddress)(const char *function_name) CL_EXT_SUFFIX__VERSION_1_1_DEPRECATED; - -// GL and other APIs -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromGLBuffer)( - cl_context context, - cl_mem_flags flags, - cl_GLuint bufobj, - int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromGLTexture)( - cl_context context, - cl_mem_flags flags, - cl_GLenum target, - cl_GLint miplevel, - cl_GLuint texture, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromGLTexture2D)( - cl_context context, - cl_mem_flags flags, - cl_GLenum target, - cl_GLint miplevel, - cl_GLuint texture, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromGLTexture3D)( - cl_context context, - cl_mem_flags flags, - cl_GLenum target, - cl_GLint miplevel, - cl_GLuint texture, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromGLRenderbuffer)( - cl_context context, - cl_mem_flags flags, - cl_GLuint renderbuffer, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetGLObjectInfo)( - cl_mem memobj, - cl_gl_object_type * gl_object_type, - cl_GLuint * gl_object_name) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetGLTextureInfo)( - cl_mem memobj, - cl_gl_texture_info param_name, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueAcquireGLObjects)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueReleaseGLObjects)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -/* cl_khr_gl_sharing */ -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetGLContextInfoKHR)( - const cl_context_properties *properties, - cl_gl_context_info param_name, - size_t param_value_size, - void *param_value, - size_t *param_value_size_ret); - -/* cl_khr_gl_event */ -typedef CL_API_ENTRY cl_event (CL_API_CALL *KHRpfn_clCreateEventFromGLsyncKHR)( - cl_context context, - cl_GLsync sync, - cl_int *errcode_ret); - - -#if defined(_WIN32) - -/* cl_khr_d3d10_sharing */ - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetDeviceIDsFromD3D10KHR)( - cl_platform_id platform, - cl_d3d10_device_source_khr d3d_device_source, - void * d3d_object, - cl_d3d10_device_set_khr d3d_device_set, - cl_uint num_entries, - cl_device_id * devices, - cl_uint * num_devices) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromD3D10BufferKHR)( - cl_context context, - cl_mem_flags flags, - ID3D10Buffer * resource, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromD3D10Texture2DKHR)( - cl_context context, - cl_mem_flags flags, - ID3D10Texture2D * resource, - UINT subresource, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromD3D10Texture3DKHR)( - cl_context context, - cl_mem_flags flags, - ID3D10Texture3D * resource, - UINT subresource, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueAcquireD3D10ObjectsKHR)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueReleaseD3D10ObjectsKHR)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_0; - -extern CL_API_ENTRY cl_int CL_API_CALL -clGetDeviceIDsFromD3D10KHR( - cl_platform_id platform, - cl_d3d10_device_source_khr d3d_device_source, - void *d3d_object, - cl_d3d10_device_set_khr d3d_device_set, - cl_uint num_entries, - cl_device_id *devices, - cl_uint *num_devices); - -extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromD3D10BufferKHR( - cl_context context, - cl_mem_flags flags, - ID3D10Buffer *resource, - cl_int *errcode_ret); - -extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromD3D10Texture2DKHR( - cl_context context, - cl_mem_flags flags, - ID3D10Texture2D * resource, - UINT subresource, - cl_int * errcode_ret); - -extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromD3D10Texture3DKHR( - cl_context context, - cl_mem_flags flags, - ID3D10Texture3D *resource, - UINT subresource, - cl_int *errcode_ret); - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueAcquireD3D10ObjectsKHR( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem *mem_objects, - cl_uint num_events_in_wait_list, - const cl_event *event_wait_list, - cl_event *event); - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReleaseD3D10ObjectsKHR( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem *mem_objects, - cl_uint num_events_in_wait_list, - const cl_event *event_wait_list, - cl_event *event); - -/* cl_khr_d3d11_sharing */ -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetDeviceIDsFromD3D11KHR)( - cl_platform_id platform, - cl_d3d11_device_source_khr d3d_device_source, - void * d3d_object, - cl_d3d11_device_set_khr d3d_device_set, - cl_uint num_entries, - cl_device_id * devices, - cl_uint * num_devices) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromD3D11BufferKHR)( - cl_context context, - cl_mem_flags flags, - ID3D11Buffer * resource, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromD3D11Texture2DKHR)( - cl_context context, - cl_mem_flags flags, - ID3D11Texture2D * resource, - UINT subresource, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromD3D11Texture3DKHR)( - cl_context context, - cl_mem_flags flags, - ID3D11Texture3D * resource, - UINT subresource, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueAcquireD3D11ObjectsKHR)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueReleaseD3D11ObjectsKHR)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -/* cl_khr_dx9_media_sharing */ -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetDeviceIDsFromDX9MediaAdapterKHR)( - cl_platform_id platform, - cl_uint num_media_adapters, - cl_dx9_media_adapter_type_khr * media_adapters_type, - void * media_adapters, - cl_dx9_media_adapter_set_khr media_adapter_set, - cl_uint num_entries, - cl_device_id * devices, - cl_uint * num_devices) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromDX9MediaSurfaceKHR)( - cl_context context, - cl_mem_flags flags, - cl_dx9_media_adapter_type_khr adapter_type, - void * surface_info, - cl_uint plane, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueAcquireDX9MediaSurfacesKHR)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueReleaseDX9MediaSurfacesKHR)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_1_2; - -/* cl_khr_d3d11_sharing */ -extern CL_API_ENTRY cl_int CL_API_CALL -clGetDeviceIDsFromD3D11KHR( - cl_platform_id platform, - cl_d3d11_device_source_khr d3d_device_source, - void * d3d_object, - cl_d3d11_device_set_khr d3d_device_set, - cl_uint num_entries, - cl_device_id * devices, - cl_uint * num_devices); - -extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromD3D11BufferKHR( - cl_context context, - cl_mem_flags flags, - ID3D11Buffer * resource, - cl_int * errcode_ret); - -extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromD3D11Texture2DKHR( - cl_context context, - cl_mem_flags flags, - ID3D11Texture2D * resource, - UINT subresource, - cl_int * errcode_ret); - -extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromD3D11Texture3DKHR( - cl_context context, - cl_mem_flags flags, - ID3D11Texture3D * resource, - UINT subresource, - cl_int * errcode_ret); - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueAcquireD3D11ObjectsKHR( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event); - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReleaseD3D11ObjectsKHR( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event); - -/* cl_khr_dx9_media_sharing */ -extern CL_API_ENTRY cl_int CL_API_CALL -clGetDeviceIDsFromDX9MediaAdapterKHR( - cl_platform_id platform, - cl_uint num_media_adapters, - cl_dx9_media_adapter_type_khr * media_adapter_type, - void * media_adapters, - cl_dx9_media_adapter_set_khr media_adapter_set, - cl_uint num_entries, - cl_device_id * devices, - cl_uint * num_devices); - -extern CL_API_ENTRY cl_mem CL_API_CALL -clCreateFromDX9MediaSurfaceKHR( - cl_context context, - cl_mem_flags flags, - cl_dx9_media_adapter_type_khr adapter_type, - void * surface_info, - cl_uint plane, - cl_int * errcode_ret); - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueAcquireDX9MediaSurfacesKHR( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event); - -extern CL_API_ENTRY cl_int CL_API_CALL -clEnqueueReleaseDX9MediaSurfacesKHR( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem * mem_objects, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event); - -#else - -/* cl_khr_d3d10_sharing */ -typedef void *KHRpfn_clGetDeviceIDsFromD3D10KHR; -typedef void *KHRpfn_clCreateFromD3D10BufferKHR; -typedef void *KHRpfn_clCreateFromD3D10Texture2DKHR; -typedef void *KHRpfn_clCreateFromD3D10Texture3DKHR; -typedef void *KHRpfn_clEnqueueAcquireD3D10ObjectsKHR; -typedef void *KHRpfn_clEnqueueReleaseD3D10ObjectsKHR; - -/* cl_khr_d3d11_sharing */ -typedef void *KHRpfn_clGetDeviceIDsFromD3D11KHR; -typedef void *KHRpfn_clCreateFromD3D11BufferKHR; -typedef void *KHRpfn_clCreateFromD3D11Texture2DKHR; -typedef void *KHRpfn_clCreateFromD3D11Texture3DKHR; -typedef void *KHRpfn_clEnqueueAcquireD3D11ObjectsKHR; -typedef void *KHRpfn_clEnqueueReleaseD3D11ObjectsKHR; - -/* cl_khr_dx9_media_sharing */ -typedef void *KHRpfn_clCreateFromDX9MediaSurfaceKHR; -typedef void *KHRpfn_clEnqueueAcquireDX9MediaSurfacesKHR; -typedef void *KHRpfn_clEnqueueReleaseDX9MediaSurfacesKHR; -typedef void *KHRpfn_clGetDeviceIDsFromDX9MediaAdapterKHR; - -#endif - -/* OpenCL 1.1 */ - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetEventCallback)( - cl_event /* event */, - cl_int /* command_exec_callback_type */, - void (CL_CALLBACK * /* pfn_notify */)(cl_event, cl_int, void *), - void * /* user_data */) CL_API_SUFFIX__VERSION_1_1; - -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateSubBuffer)( - cl_mem /* buffer */, - cl_mem_flags /* flags */, - cl_buffer_create_type /* buffer_create_type */, - const void * /* buffer_create_info */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetMemObjectDestructorCallback)( - cl_mem /* memobj */, - void (CL_CALLBACK * /*pfn_notify*/)( cl_mem /* memobj */, void* /*user_data*/), - void * /*user_data */ ) CL_API_SUFFIX__VERSION_1_1; - -typedef CL_API_ENTRY cl_event (CL_API_CALL *KHRpfn_clCreateUserEvent)( - cl_context /* context */, - cl_int * /* errcode_ret */) CL_API_SUFFIX__VERSION_1_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetUserEventStatus)( - cl_event /* event */, - cl_int /* execution_status */) CL_API_SUFFIX__VERSION_1_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clCreateSubDevicesEXT)( - cl_device_id in_device, - const cl_device_partition_property_ext * partition_properties, - cl_uint num_entries, - cl_device_id * out_devices, - cl_uint * num_devices); - -typedef CL_API_ENTRY cl_int (CL_API_CALL * KHRpfn_clRetainDeviceEXT)( - cl_device_id device) CL_API_SUFFIX__VERSION_1_0; - -typedef CL_API_ENTRY cl_int (CL_API_CALL * KHRpfn_clReleaseDeviceEXT)( - cl_device_id device) CL_API_SUFFIX__VERSION_1_0; - -/* cl_khr_egl_image */ -typedef CL_API_ENTRY cl_mem (CL_API_CALL *KHRpfn_clCreateFromEGLImageKHR)( - cl_context context, - CLeglDisplayKHR display, - CLeglImageKHR image, - cl_mem_flags flags, - const cl_egl_image_properties_khr *properties, - cl_int *errcode_ret); - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueAcquireEGLObjectsKHR)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem *mem_objects, - cl_uint num_events_in_wait_list, - const cl_event *event_wait_list, - cl_event *event); - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueReleaseEGLObjectsKHR)( - cl_command_queue command_queue, - cl_uint num_objects, - const cl_mem *mem_objects, - cl_uint num_events_in_wait_list, - const cl_event *event_wait_list, - cl_event *event); - -/* cl_khr_egl_event */ -typedef CL_API_ENTRY cl_event (CL_API_CALL *KHRpfn_clCreateEventFromEGLSyncKHR)( - cl_context context, - CLeglSyncKHR sync, - CLeglDisplayKHR display, - cl_int *errcode_ret); - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clSetDefaultDeviceCommandQueue)( - cl_context context, - cl_device_id device, - cl_command_queue command_queue) CL_API_SUFFIX__VERSION_2_1; - -typedef CL_API_ENTRY cl_program (CL_API_CALL *KHRpfn_clCreateProgramWithIL)( - cl_context context, - const void * il, - size_t length, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_2_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetKernelSubGroupInfo )( - cl_kernel kernel, - cl_device_id device, - cl_kernel_sub_group_info param_name, - size_t input_value_size, - const void * input_value, - size_t param_value_size, - void * param_value, - size_t * param_value_size_ret) CL_API_SUFFIX__VERSION_2_1; - -typedef CL_API_ENTRY cl_kernel (CL_API_CALL *KHRpfn_clCloneKernel)( - cl_kernel source_kernel, - cl_int * errcode_ret) CL_API_SUFFIX__VERSION_2_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clEnqueueSVMMigrateMem)( - cl_command_queue command_queue, - cl_uint num_svm_pointers, - const void ** svm_pointers, - const size_t * sizes, - cl_mem_migration_flags flags, - cl_uint num_events_in_wait_list, - const cl_event * event_wait_list, - cl_event * event) CL_API_SUFFIX__VERSION_2_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetDeviceAndHostTimer)( - cl_device_id device, - cl_ulong * device_timestamp, - cl_ulong * host_timestamp) CL_API_SUFFIX__VERSION_2_1; - -typedef CL_API_ENTRY cl_int (CL_API_CALL *KHRpfn_clGetHostTimer)( - cl_device_id device, - cl_ulong * host_timestamp) CL_API_SUFFIX__VERSION_2_1; - -/* - * - * vendor dispatch table structure - * - * note that the types in the structure KHRicdVendorDispatch mirror the function - * names listed in the string table khrIcdVendorDispatchFunctionNames - * - */ - -typedef struct KHRicdVendorDispatchRec KHRicdVendorDispatch; - -struct KHRicdVendorDispatchRec -{ - /* OpenCL 1.0 */ - KHRpfn_clGetPlatformIDs clGetPlatformIDs; - KHRpfn_clGetPlatformInfo clGetPlatformInfo; - KHRpfn_clGetDeviceIDs clGetDeviceIDs; - KHRpfn_clGetDeviceInfo clGetDeviceInfo; - KHRpfn_clCreateContext clCreateContext; - KHRpfn_clCreateContextFromType clCreateContextFromType; - KHRpfn_clRetainContext clRetainContext; - KHRpfn_clReleaseContext clReleaseContext; - KHRpfn_clGetContextInfo clGetContextInfo; - KHRpfn_clCreateCommandQueue clCreateCommandQueue; - KHRpfn_clRetainCommandQueue clRetainCommandQueue; - KHRpfn_clReleaseCommandQueue clReleaseCommandQueue; - KHRpfn_clGetCommandQueueInfo clGetCommandQueueInfo; - KHRpfn_clSetCommandQueueProperty clSetCommandQueueProperty; - KHRpfn_clCreateBuffer clCreateBuffer; - KHRpfn_clCreateImage2D clCreateImage2D; - KHRpfn_clCreateImage3D clCreateImage3D; - KHRpfn_clRetainMemObject clRetainMemObject; - KHRpfn_clReleaseMemObject clReleaseMemObject; - KHRpfn_clGetSupportedImageFormats clGetSupportedImageFormats; - KHRpfn_clGetMemObjectInfo clGetMemObjectInfo; - KHRpfn_clGetImageInfo clGetImageInfo; - KHRpfn_clCreateSampler clCreateSampler; - KHRpfn_clRetainSampler clRetainSampler; - KHRpfn_clReleaseSampler clReleaseSampler; - KHRpfn_clGetSamplerInfo clGetSamplerInfo; - KHRpfn_clCreateProgramWithSource clCreateProgramWithSource; - KHRpfn_clCreateProgramWithBinary clCreateProgramWithBinary; - KHRpfn_clRetainProgram clRetainProgram; - KHRpfn_clReleaseProgram clReleaseProgram; - KHRpfn_clBuildProgram clBuildProgram; - KHRpfn_clUnloadCompiler clUnloadCompiler; - KHRpfn_clGetProgramInfo clGetProgramInfo; - KHRpfn_clGetProgramBuildInfo clGetProgramBuildInfo; - KHRpfn_clCreateKernel clCreateKernel; - KHRpfn_clCreateKernelsInProgram clCreateKernelsInProgram; - KHRpfn_clRetainKernel clRetainKernel; - KHRpfn_clReleaseKernel clReleaseKernel; - KHRpfn_clSetKernelArg clSetKernelArg; - KHRpfn_clGetKernelInfo clGetKernelInfo; - KHRpfn_clGetKernelWorkGroupInfo clGetKernelWorkGroupInfo; - KHRpfn_clWaitForEvents clWaitForEvents; - KHRpfn_clGetEventInfo clGetEventInfo; - KHRpfn_clRetainEvent clRetainEvent; - KHRpfn_clReleaseEvent clReleaseEvent; - KHRpfn_clGetEventProfilingInfo clGetEventProfilingInfo; - KHRpfn_clFlush clFlush; - KHRpfn_clFinish clFinish; - KHRpfn_clEnqueueReadBuffer clEnqueueReadBuffer; - KHRpfn_clEnqueueWriteBuffer clEnqueueWriteBuffer; - KHRpfn_clEnqueueCopyBuffer clEnqueueCopyBuffer; - KHRpfn_clEnqueueReadImage clEnqueueReadImage; - KHRpfn_clEnqueueWriteImage clEnqueueWriteImage; - KHRpfn_clEnqueueCopyImage clEnqueueCopyImage; - KHRpfn_clEnqueueCopyImageToBuffer clEnqueueCopyImageToBuffer; - KHRpfn_clEnqueueCopyBufferToImage clEnqueueCopyBufferToImage; - KHRpfn_clEnqueueMapBuffer clEnqueueMapBuffer; - KHRpfn_clEnqueueMapImage clEnqueueMapImage; - KHRpfn_clEnqueueUnmapMemObject clEnqueueUnmapMemObject; - KHRpfn_clEnqueueNDRangeKernel clEnqueueNDRangeKernel; - KHRpfn_clEnqueueTask clEnqueueTask; - KHRpfn_clEnqueueNativeKernel clEnqueueNativeKernel; - KHRpfn_clEnqueueMarker clEnqueueMarker; - KHRpfn_clEnqueueWaitForEvents clEnqueueWaitForEvents; - KHRpfn_clEnqueueBarrier clEnqueueBarrier; - KHRpfn_clGetExtensionFunctionAddress clGetExtensionFunctionAddress; - KHRpfn_clCreateFromGLBuffer clCreateFromGLBuffer; - KHRpfn_clCreateFromGLTexture2D clCreateFromGLTexture2D; - KHRpfn_clCreateFromGLTexture3D clCreateFromGLTexture3D; - KHRpfn_clCreateFromGLRenderbuffer clCreateFromGLRenderbuffer; - KHRpfn_clGetGLObjectInfo clGetGLObjectInfo; - KHRpfn_clGetGLTextureInfo clGetGLTextureInfo; - KHRpfn_clEnqueueAcquireGLObjects clEnqueueAcquireGLObjects; - KHRpfn_clEnqueueReleaseGLObjects clEnqueueReleaseGLObjects; - KHRpfn_clGetGLContextInfoKHR clGetGLContextInfoKHR; - - /* cl_khr_d3d10_sharing */ - KHRpfn_clGetDeviceIDsFromD3D10KHR clGetDeviceIDsFromD3D10KHR; - KHRpfn_clCreateFromD3D10BufferKHR clCreateFromD3D10BufferKHR; - KHRpfn_clCreateFromD3D10Texture2DKHR clCreateFromD3D10Texture2DKHR; - KHRpfn_clCreateFromD3D10Texture3DKHR clCreateFromD3D10Texture3DKHR; - KHRpfn_clEnqueueAcquireD3D10ObjectsKHR clEnqueueAcquireD3D10ObjectsKHR; - KHRpfn_clEnqueueReleaseD3D10ObjectsKHR clEnqueueReleaseD3D10ObjectsKHR; - - /* OpenCL 1.1 */ - KHRpfn_clSetEventCallback clSetEventCallback; - KHRpfn_clCreateSubBuffer clCreateSubBuffer; - KHRpfn_clSetMemObjectDestructorCallback clSetMemObjectDestructorCallback; - KHRpfn_clCreateUserEvent clCreateUserEvent; - KHRpfn_clSetUserEventStatus clSetUserEventStatus; - KHRpfn_clEnqueueReadBufferRect clEnqueueReadBufferRect; - KHRpfn_clEnqueueWriteBufferRect clEnqueueWriteBufferRect; - KHRpfn_clEnqueueCopyBufferRect clEnqueueCopyBufferRect; - - /* cl_ext_device_fission */ - KHRpfn_clCreateSubDevicesEXT clCreateSubDevicesEXT; - KHRpfn_clRetainDeviceEXT clRetainDeviceEXT; - KHRpfn_clReleaseDeviceEXT clReleaseDeviceEXT; - - /* cl_khr_gl_event */ - KHRpfn_clCreateEventFromGLsyncKHR clCreateEventFromGLsyncKHR; - - /* OpenCL 1.2 */ - KHRpfn_clCreateSubDevices clCreateSubDevices; - KHRpfn_clRetainDevice clRetainDevice; - KHRpfn_clReleaseDevice clReleaseDevice; - KHRpfn_clCreateImage clCreateImage; - KHRpfn_clCreateProgramWithBuiltInKernels clCreateProgramWithBuiltInKernels; - KHRpfn_clCompileProgram clCompileProgram; - KHRpfn_clLinkProgram clLinkProgram; - KHRpfn_clUnloadPlatformCompiler clUnloadPlatformCompiler; - KHRpfn_clGetKernelArgInfo clGetKernelArgInfo; - KHRpfn_clEnqueueFillBuffer clEnqueueFillBuffer; - KHRpfn_clEnqueueFillImage clEnqueueFillImage; - KHRpfn_clEnqueueMigrateMemObjects clEnqueueMigrateMemObjects; - KHRpfn_clEnqueueMarkerWithWaitList clEnqueueMarkerWithWaitList; - KHRpfn_clEnqueueBarrierWithWaitList clEnqueueBarrierWithWaitList; - KHRpfn_clGetExtensionFunctionAddressForPlatform clGetExtensionFunctionAddressForPlatform; - KHRpfn_clCreateFromGLTexture clCreateFromGLTexture; - - /* cl_khr_d3d11_sharing */ - KHRpfn_clGetDeviceIDsFromD3D11KHR clGetDeviceIDsFromD3D11KHR; - KHRpfn_clCreateFromD3D11BufferKHR clCreateFromD3D11BufferKHR; - KHRpfn_clCreateFromD3D11Texture2DKHR clCreateFromD3D11Texture2DKHR; - KHRpfn_clCreateFromD3D11Texture3DKHR clCreateFromD3D11Texture3DKHR; - KHRpfn_clCreateFromDX9MediaSurfaceKHR clCreateFromDX9MediaSurfaceKHR; - KHRpfn_clEnqueueAcquireD3D11ObjectsKHR clEnqueueAcquireD3D11ObjectsKHR; - KHRpfn_clEnqueueReleaseD3D11ObjectsKHR clEnqueueReleaseD3D11ObjectsKHR; - - /* cl_khr_dx9_media_sharing */ - KHRpfn_clGetDeviceIDsFromDX9MediaAdapterKHR clGetDeviceIDsFromDX9MediaAdapterKHR; - KHRpfn_clEnqueueAcquireDX9MediaSurfacesKHR clEnqueueAcquireDX9MediaSurfacesKHR; - KHRpfn_clEnqueueReleaseDX9MediaSurfacesKHR clEnqueueReleaseDX9MediaSurfacesKHR; - - /* cl_khr_egl_image */ - KHRpfn_clCreateFromEGLImageKHR clCreateFromEGLImageKHR; - KHRpfn_clEnqueueAcquireEGLObjectsKHR clEnqueueAcquireEGLObjectsKHR; - KHRpfn_clEnqueueReleaseEGLObjectsKHR clEnqueueReleaseEGLObjectsKHR; - - /* cl_khr_egl_event */ - KHRpfn_clCreateEventFromEGLSyncKHR clCreateEventFromEGLSyncKHR; - - /* OpenCL 2.0 */ - KHRpfn_clCreateCommandQueueWithProperties clCreateCommandQueueWithProperties; - KHRpfn_clCreatePipe clCreatePipe; - KHRpfn_clGetPipeInfo clGetPipeInfo; - KHRpfn_clSVMAlloc clSVMAlloc; - KHRpfn_clSVMFree clSVMFree; - KHRpfn_clEnqueueSVMFree clEnqueueSVMFree; - KHRpfn_clEnqueueSVMMemcpy clEnqueueSVMMemcpy; - KHRpfn_clEnqueueSVMMemFill clEnqueueSVMMemFill; - KHRpfn_clEnqueueSVMMap clEnqueueSVMMap; - KHRpfn_clEnqueueSVMUnmap clEnqueueSVMUnmap; - KHRpfn_clCreateSamplerWithProperties clCreateSamplerWithProperties; - KHRpfn_clSetKernelArgSVMPointer clSetKernelArgSVMPointer; - KHRpfn_clSetKernelExecInfo clSetKernelExecInfo; - - /* cl_khr_sub_groups */ - KHRpfn_clGetKernelSubGroupInfoKHR clGetKernelSubGroupInfoKHR; - - /* OpenCL 2.1 */ - KHRpfn_clCloneKernel clCloneKernel; - KHRpfn_clCreateProgramWithIL clCreateProgramWithIL; - KHRpfn_clEnqueueSVMMigrateMem clEnqueueSVMMigrateMem; - KHRpfn_clGetDeviceAndHostTimer clGetDeviceAndHostTimer; - KHRpfn_clGetHostTimer clGetHostTimer; - KHRpfn_clGetKernelSubGroupInfo clGetKernelSubGroupInfo; - KHRpfn_clSetDefaultDeviceCommandQueue clSetDefaultDeviceCommandQueue; - - /* OpenCL 2.2 */ - KHRpfn_clSetProgramReleaseCallback clSetProgramReleaseCallback; - KHRpfn_clSetProgramSpecializationConstant clSetProgramSpecializationConstant; -}; +#include /* * @@ -1464,47 +61,47 @@ struct KHRicdVendorDispatchRec struct _cl_platform_id { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; struct _cl_device_id { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; struct _cl_context { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; struct _cl_command_queue { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; struct _cl_mem { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; struct _cl_program { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; struct _cl_kernel { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; struct _cl_event { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; struct _cl_sampler { - KHRicdVendorDispatch *dispatch; + cl_icd_dispatch *dispatch; }; #endif // _ICD_DISPATCH_H_ diff --git a/vdi/CMakeLists.txt b/vdi/CMakeLists.txt index 0bf319be47..ee772874af 100644 --- a/vdi/CMakeLists.txt +++ b/vdi/CMakeLists.txt @@ -13,7 +13,7 @@ set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(LIB_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib) set(CONFIG_PACKAGE_INSTALL_DIR ${LIB_INSTALL_DIR}/cmake/hip) -add_definitions(-D__HIP_VDI__ -D__HIP_PLATFORM_HCC__ -DLINUX -D__x86_64__ -D__AMD64__ -DUNIX_OS -DqLittleEndian -DOPENCL_MAJOR=2 -DOPENCL_MINOR=0 -DWITH_AQL -DWITH_ONLINE_COMPILER -DATI_OS_LINUX -DATI_ARCH_X86 -DLITTLEENDIAN_CPU -DATI_BITS_64 -DATI_COMP_GCC -DWITH_HSA_DEVICE -DWITH_TARGET_AMDGCN -DOPENCL_EXPORTS -DCL_USE_DEPRECATED_OPENCL_1_0_APIS -DCL_USE_DEPRECATED_OPENCL_1_1_APIS -DCL_USE_DEPRECATED_OPENCL_1_2_APIS -DCL_USE_DEPRECATED_OPENCL_2_0_APIS -DVEGA10_ONLY=false -DWITH_LIGHTNING_COMPILER -DUSE_PROF_API) +add_definitions(-D__HIP_VDI__ -D__HIP_PLATFORM_HCC__ -DLINUX -D__x86_64__ -D__AMD64__ -DUNIX_OS -DqLittleEndian -DOPENCL_MAJOR=2 -DOPENCL_MINOR=0 -DCL_TARGET_OPENCL_VERSION=220 -DWITH_AQL -DWITH_ONLINE_COMPILER -DATI_OS_LINUX -DATI_ARCH_X86 -DLITTLEENDIAN_CPU -DATI_BITS_64 -DATI_COMP_GCC -DWITH_HSA_DEVICE -DWITH_TARGET_AMDGCN -DOPENCL_EXPORTS -DCL_USE_DEPRECATED_OPENCL_1_0_APIS -DCL_USE_DEPRECATED_OPENCL_1_1_APIS -DCL_USE_DEPRECATED_OPENCL_1_2_APIS -DCL_USE_DEPRECATED_OPENCL_2_0_APIS -DVEGA10_ONLY=false -DWITH_LIGHTNING_COMPILER -DUSE_PROF_API) if(CMAKE_BUILD_TYPE MATCHES "^Debug$") add_definitions(-DDEBUG) diff --git a/vdi/fixme.cpp b/vdi/fixme.cpp index 0319226bc3..90f034f63e 100644 --- a/vdi/fixme.cpp +++ b/vdi/fixme.cpp @@ -21,7 +21,7 @@ #include "vdi_common.hpp" #include -KHRicdVendorDispatch amd::ICDDispatchedObject::icdVendorDispatch_[] = {0}; +cl_icd_dispatch amd::ICDDispatchedObject::icdVendorDispatch_[] = {0}; amd::PlatformIDS amd::PlatformID::Platform = {amd::ICDDispatchedObject::icdVendorDispatch_}; RUNTIME_ENTRY(cl_int, clGetDeviceIDs, From d967be8eeb93e300ae602a614f9a2a7305a9c729 Mon Sep 17 00:00:00 2001 From: Tao Sang Date: Wed, 25 Mar 2020 18:47:28 -0400 Subject: [PATCH 137/154] Support hipFuncGetAttributes with hip-clang+Hcc RT Fix issues of missing kernel function symbols and missing argument list via using __hipRegister* functions. Then the following tests can pass, directed_tests/runtimeApi/module/hipFuncGetAttributes directed_tests/runtimeApi/module/hipExtLaunchMultiKernelMultiDevice directed_tests/gcc/LaunchKernel Change-Id: I52135b61e8283eb4f9f10f77895151e4e55418d9 --- src/hip_clang.cpp | 44 +++++++++++++++++++++++++++++++++--- src/hip_hcc_internal.h | 12 +++++++++- src/hip_module.cpp | 2 ++ src/program_state.inl | 14 ++++++++---- tests/src/gcc/LaunchKernel.c | 8 +++---- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/hip_clang.cpp b/src/hip_clang.cpp index 635aa8391f..e8f3e86881 100644 --- a/src/hip_clang.cpp +++ b/src/hip_clang.cpp @@ -28,6 +28,7 @@ THE SOFTWARE. #include "hip_hcc_internal.h" #include "hip_fatbin.h" #include "trace_helper.h" +#include "program_state.inl" #ifdef __GNUC__ #pragma GCC visibility push (default) @@ -94,8 +95,10 @@ __hipRegisterFatBinary(const void* data) agent); if (module->executable.handle) { - modules->at(deviceId) = module; - tprintf(DB_FB, "Loaded code object for %s\n", name); + hip_impl::program_state_impl::read_kernarg_metadata(image, module->kernargs); + modules->at(deviceId) = module; + + tprintf(DB_FB, "Loaded code object for %s, args size=%ld\n", name, module->kernargs.size()); } else { fprintf(stderr, "Failed to load code object for %s\n", name); abort(); @@ -232,7 +235,7 @@ static DeviceVar* findVar(std::string hostVar, int deviceId, hipModule_t hmod) { } hipError_t ihipGetGlobalVar(hipDeviceptr_t* dev_ptr, size_t* size_ptr, - const char* hostVar, hipModule_t hmod) { + const char* hostVar, hipModule_t hmod) { GET_TLS(); auto ctx = ihipGetTlsDefaultCtx(); @@ -425,6 +428,41 @@ extern "C" hipError_t __hipPopCallConfiguration( return hipSuccess; } +int getCurrentDeviceId() +{ + GET_TLS(); + + int deviceId = 0; + auto ctx = ihipGetTlsDefaultCtx(); + + if(!ctx) return deviceId; + + LockedAccessor_CtxCrit_t crit(ctx->criticalData()); + + if(crit->_execStack.size() != 0) + { + auto &exec = crit->_execStack.top(); + + if (exec._hStream) { + deviceId = exec._hStream->getDevice()->_deviceId; + } else if (ctx->getDevice()) { + deviceId = ctx->getDevice()->_deviceId; + } + } else if (ctx->getDevice()) { + deviceId = ctx->getDevice()->_deviceId; + } + return deviceId; +} + +hipFunction_t ihipGetDeviceFunction(const void *hostFunction) +{ + int deviceId = getCurrentDeviceId(); + auto it = g_functions.find(hostFunction); + if (it == g_functions.end() || !it->second[deviceId]) { + return nullptr; + } + return it->second[deviceId]; +} hipError_t hipSetupArgument( const void *arg, diff --git a/src/hip_hcc_internal.h b/src/hip_hcc_internal.h index 8eec7292c2..104fd910a8 100644 --- a/src/hip_hcc_internal.h +++ b/src/hip_hcc_internal.h @@ -33,7 +33,7 @@ THE SOFTWARE. #include "hip_prof_api.h" #include "hip_util.h" #include "env.h" - +#include #if (__hcc_workweek__ < 16354) #error("This version of HIP requires a newer version of HCC."); @@ -1079,4 +1079,14 @@ static inline ihipCtx_t* iihipGetTlsDefaultCtx(TlsData* tls) { return tls->defaultCtx; } +/** + * @brief Get device function from host kernel function pointer + * Needed only for clang + HIP-HCC RT + * + * @param [in] hostFunction host kernel function pointer + * + * @returns hipFuntion_t, nullptr + */ +hipFunction_t ihipGetDeviceFunction(const void *hostFunction); + #endif diff --git a/src/hip_module.cpp b/src/hip_module.cpp index b6fbc45866..0a7348a3a2 100644 --- a/src/hip_module.cpp +++ b/src/hip_module.cpp @@ -344,6 +344,8 @@ hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList free(kds); return hipErrorInvalidValue; } + if (!kds[i]->_kernarg_layout.empty()) continue; + hip_impl::kernargs_size_align kargs = ps.get_kernargs_size_align( reinterpret_cast(lp.func)); kds[i]->_kernarg_layout = *reinterpret_cast>*>( diff --git a/src/program_state.inl b/src/program_state.inl index d77abd9674..548a56795f 100644 --- a/src/program_state.inl +++ b/src/program_state.inl @@ -19,6 +19,8 @@ #include #include #include "hc.hpp" +#include "hip_hcc_internal.h" +#include "trace_helper.h" #include @@ -944,14 +946,16 @@ public: auto it0 = get_functions(agent).find(function_address); - if (it0 == get_functions(agent).cend()) { - hip_throw(std::runtime_error{ + if (it0 != get_functions(agent).cend()) return it0->second; + + // For hip-clang compiler + Hcc RT + hipFunction_t f = ihipGetDeviceFunction((const void*)function_address); + if (f) return reinterpret_cast(*f); + + hip_throw(std::runtime_error{ "No device code available for function: " + std::string(name(function_address)) + ", for agent: " + name(agent)}); - } - - return it0->second; } const std::vector>& diff --git a/tests/src/gcc/LaunchKernel.c b/tests/src/gcc/LaunchKernel.c index ef910a423e..d2fc854510 100644 --- a/tests/src/gcc/LaunchKernel.c +++ b/tests/src/gcc/LaunchKernel.c @@ -19,10 +19,10 @@ /* HIT_START - * BUILD_CMD: gpu.o %hc -I%hip-path/include -g -c %S/gpu.cpp -o %T/gpu.o EXCLUDE_HIP_PLATFORM nvcc vdi EXCLUDE_HIP_COMPILER clang - * BUILD_CMD: launchkernel.o %hc -D__HIP_PLATFORM_HCC__ -g -I%hip-path/include -c %S/LaunchKernel.c -o %T/launchkernel.o EXCLUDE_HIP_PLATFORM nvcc vdi EXCLUDE_HIP_COMPILER clang - * BUILD_CMD: LaunchKernel %hc %T/launchkernel.o %T/gpu.o -g -Wl,--rpath=%hip-path/lib %hip-path/lib/libhip_hcc.so -o %T/%t DEPENDS gpu.o launchkernel.o EXCLUDE_HIP_PLATFORM nvcc vdi EXCLUDE_HIP_COMPILER clang - * TEST: %t EXCLUDE_HIP_PLATFORM nvcc vdi EXCLUDE_HIP_COMPILER clang + * BUILD_CMD: gpu.o %hc -I%hip-path/include -g -c %S/gpu.cpp -o %T/gpu.o EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD_CMD: launchkernel.o %hc -D__HIP_PLATFORM_HCC__ -g -I%hip-path/include -c %S/LaunchKernel.c -o %T/launchkernel.o EXCLUDE_HIP_PLATFORM nvcc vdi + * BUILD_CMD: LaunchKernel %hc %T/launchkernel.o %T/gpu.o -g -Wl,--rpath=%hip-path/lib %hip-path/lib/libhip_hcc.so -o %T/%t DEPENDS gpu.o launchkernel.o EXCLUDE_HIP_PLATFORM nvcc vdi + * TEST: %t EXCLUDE_HIP_PLATFORM nvcc vdi * HIT_END */ From 178fc17846bfe209428759b33e9976baaa89bda5 Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Fri, 27 Mar 2020 11:32:46 -0700 Subject: [PATCH 138/154] Check event status before notify Change-Id: I68f6bbbf236e49b859be2d5afbe0c8282fe15dd3 --- vdi/hip_event.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vdi/hip_event.cpp b/vdi/hip_event.cpp index 5c9d570ea1..03498c47cf 100644 --- a/vdi/hip_event.cpp +++ b/vdi/hip_event.cpp @@ -25,7 +25,9 @@ namespace hip { bool Event::ready() { - event_->notifyCmdQueue(); + if (event_->status() != CL_COMPLETE) { + event_->notifyCmdQueue(); + } return (event_->status() == CL_COMPLETE); } From a91b82f00e761875e735b0025645fc652149611b Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Thu, 26 Mar 2020 12:42:55 -0400 Subject: [PATCH 139/154] (SWDEV-228782) The only requirment from hipMallocPitch() is that the returned pitch is aligned to the HW image pitch alignment. There is no restriction on the size of the allocation, since the memory might not be used for images. Change-Id: I97438e5fe4012ca4721b14b85f514dbac803c17c --- vdi/hip_memory.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 2ffa356907..80710e0d13 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -358,17 +358,13 @@ hipError_t ihipMallocPitch(void** ptr, size_t* pitch, size_t width, size_t heigh amd::Device* device = hip::getCurrentDevice()->devices()[0]; - if ((width == 0) || (height == 0)) { - *ptr = nullptr; - return hipSuccess; - } - else if (!(device->info().image2DMaxWidth_ >= width && - device->info().image2DMaxHeight_ >= height ) || (ptr == nullptr)) { + if (ptr == nullptr) { return hipErrorInvalidValue; } - if (device->info().maxMemAllocSize_ < (width * height)) { - return hipErrorOutOfMemory; + if ((width == 0) || (height == 0)) { + *ptr = nullptr; + return hipSuccess; } const amd::Image::Format imageFormat(*image_format); @@ -376,6 +372,11 @@ hipError_t ihipMallocPitch(void** ptr, size_t* pitch, size_t width, size_t heigh *pitch = amd::alignUp(width * imageFormat.getElementSize(), device->info().imagePitchAlignment_); size_t sizeBytes = *pitch * height * depth; + + if (device->info().maxMemAllocSize_ < sizeBytes) { + return hipErrorOutOfMemory; + } + *ptr = amd::SvmBuffer::malloc(*hip::getCurrentDevice()->asContext(), 0, sizeBytes, device->info().memBaseAddrAlign_); From fd7a8f036731193f5ff4c5cff39fa7cbf9db1d57 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 27 Mar 2020 14:02:46 -0400 Subject: [PATCH 140/154] (SWDEV-228794) Add hipMallocHost() Change-Id: Ia3c7c5ca94b39fe30f3a51d1b60782d3472259ff --- vdi/hip_hcc.def.in | 1 + vdi/hip_hcc.map.in | 1 + vdi/hip_memory.cpp | 11 +++++++++++ 3 files changed, 13 insertions(+) diff --git a/vdi/hip_hcc.def.in b/vdi/hip_hcc.def.in index 82d0283e1a..6bf962a084 100644 --- a/vdi/hip_hcc.def.in +++ b/vdi/hip_hcc.def.in @@ -237,3 +237,4 @@ hipMipmappedArrayDestroy hipFreeMipmappedArray hipMipmappedArrayGetLevel hipGetMipmappedArrayLevel +hipMallocHost diff --git a/vdi/hip_hcc.map.in b/vdi/hip_hcc.map.in index 210789e420..ec1c8a4558 100644 --- a/vdi/hip_hcc.map.in +++ b/vdi/hip_hcc.map.in @@ -231,6 +231,7 @@ global: hipFreeMipmappedArray; hipMipmappedArrayGetLevel; hipGetMipmappedArrayLevel; + hipMallocHost; extern "C++" { hip_impl::hipLaunchKernelGGLImpl*; hip_impl::demangle*; diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 80710e0d13..467f996952 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -2101,3 +2101,14 @@ hipError_t hipGetMipmappedArrayLevel(hipArray_t *levelArray, HIP_RETURN(hipErrorNotSupported); } + +hipError_t hipMallocHost(void** ptr, + size_t size) { + HIP_INIT_API(hipMallocHost, ptr, size); + + if (ptr == nullptr) { + HIP_RETURN(hipErrorInvalidValue); + } + + HIP_RETURN(ihipMalloc(ptr, size, CL_MEM_SVM_FINE_GRAIN_BUFFER)); +} \ No newline at end of file From 6ed73f50f722248ac64d006b7dda00c1b090b0d6 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Fri, 27 Mar 2020 15:19:56 -0400 Subject: [PATCH 141/154] (SWDEV-228794) Adjust the origin of the copy if the user passes a pointer that wasn't allocated by the runtime. Change-Id: I0aeb20195ed730857a461a53f537626ec2573fd1 --- vdi/hip_memory.cpp | 47 +++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 467f996952..3fc1a2921a 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -882,6 +882,19 @@ hipError_t hipMemcpyDtoHAsync(void* dstHost, HIP_RETURN(ihipMemcpy(dstHost, srcDevice, ByteCount, hipMemcpyDeviceToHost, *hip::getQueue(stream), true)); } +inline void adjustOrigin(amd::Coord3D &origin, + size_t offset, + size_t rowPitch, + size_t slicePitch) { + size_t zOffset = offset / (slicePitch ? slicePitch : 1); + size_t yOffset = (offset - slicePitch * zOffset) / (rowPitch ? rowPitch : 1); + size_t xOffset = (offset - slicePitch * zOffset - rowPitch * yOffset); + + static_cast(origin)[0] += xOffset; + static_cast(origin)[1] += yOffset; + static_cast(origin)[2] += zOffset; +} + hipError_t ihipMemcpyAtoD(hipArray* srcArray, void* dstDevice, amd::Coord3D srcOrigin, @@ -897,9 +910,9 @@ hipError_t ihipMemcpyAtoD(hipArray* srcArray, } amd::Image* srcImage = as_amd(srcMemObj)->asImage(); - size_t offset = 0; - amd::Memory* dstMemory = getMemoryObject(dstDevice, offset); - assert(offset != 0); + size_t dstOffset = 0; + amd::Memory* dstMemory = getMemoryObject(dstDevice, dstOffset); + adjustOrigin(dstOrigin, dstOffset, dstRowPitch, dstSlicePitch); amd::BufferRect srcRect; if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcImage->getRowPitch(), srcImage->getSlicePitch())) { @@ -955,9 +968,9 @@ hipError_t ihipMemcpyDtoA(void* srcDevice, return hipErrorInvalidValue; } - size_t offset = 0; - amd::Memory* srcMemory = getMemoryObject(srcDevice, offset); - assert(offset != 0); + size_t srcOffset = 0; + amd::Memory* srcMemory = getMemoryObject(srcDevice, srcOffset); + adjustOrigin(srcOrigin, srcOffset, srcRowPitch, srcSlicePitch); amd::Image* dstImage = as_amd(dstMemObj)->asImage(); amd::BufferRect srcRect; @@ -1013,16 +1026,18 @@ hipError_t ihipMemcpyDtoD(void* srcDevice, bool isAsync = false) { size_t srcOffset = 0; amd::Memory *srcMemory = getMemoryObject(srcDevice, srcOffset); + adjustOrigin(srcOrigin, srcOffset, srcRowPitch, srcSlicePitch); size_t dstOffset = 0; amd::Memory *dstMemory = getMemoryObject(dstDevice, dstOffset); + adjustOrigin(dstOrigin, dstOffset, dstRowPitch, dstSlicePitch); amd::BufferRect srcRect; if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { return hipErrorInvalidValue; } - amd::Coord3D srcStart(srcRect.start_ + srcOffset, 0, 0); - amd::Coord3D srcEnd(srcRect.end_ + srcOffset, 1, 1); + amd::Coord3D srcStart(srcRect.start_, 0, 0); + amd::Coord3D srcEnd(srcRect.end_, 1, 1); if (!srcMemory->validateRegion(srcStart, srcEnd)) { return hipErrorInvalidValue; } @@ -1032,8 +1047,8 @@ hipError_t ihipMemcpyDtoD(void* srcDevice, return hipErrorInvalidValue; } - amd::Coord3D dstStart(dstRect.start_ + dstOffset, 0, 0); - amd::Coord3D dstEnd(dstRect.end_ + dstOffset, 1, 1); + amd::Coord3D dstStart(dstRect.start_, 0, 0); + amd::Coord3D dstEnd(dstRect.end_, 1, 1); if (!dstMemory->validateRegion(dstStart, dstEnd)) { return hipErrorInvalidValue; } @@ -1075,14 +1090,15 @@ hipError_t ihipMemcpyDtoH(void* srcDevice, bool isAsync = false) { size_t srcOffset = 0; amd::Memory *srcMemory = getMemoryObject(srcDevice, srcOffset); + adjustOrigin(srcOrigin, srcOffset, srcRowPitch, srcSlicePitch); amd::BufferRect srcRect; if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { return hipErrorInvalidValue; } - amd::Coord3D srcStart(srcRect.start_ + srcOffset, 0, 0); - amd::Coord3D srcEnd(srcRect.end_ + srcOffset, 1, 1); + amd::Coord3D srcStart(srcRect.start_, 0, 0); + amd::Coord3D srcEnd(srcRect.end_, 1, 1); if (!srcMemory->validateRegion(srcStart, srcEnd)) { return hipErrorInvalidValue; } @@ -1128,6 +1144,7 @@ hipError_t ihipMemcpyHtoD(const void* srcHost, bool isAsync = false) { size_t dstOffset = 0; amd::Memory *dstMemory = getMemoryObject(dstDevice, dstOffset); + adjustOrigin(dstOrigin, dstOffset, dstRowPitch, dstSlicePitch); amd::BufferRect srcRect; if (!srcRect.create(static_cast(srcOrigin), static_cast(copyRegion), srcRowPitch, srcSlicePitch)) { @@ -1139,8 +1156,8 @@ hipError_t ihipMemcpyHtoD(const void* srcHost, return hipErrorInvalidValue; } - amd::Coord3D dstStart(dstRect.start_ + dstOffset, 0, 0); - amd::Coord3D dstEnd(dstRect.end_ + dstOffset, 1, 1); + amd::Coord3D dstStart(dstRect.start_, 0, 0); + amd::Coord3D dstEnd(dstRect.end_, 1, 1); if (!dstMemory->validateRegion(dstStart, dstEnd)) { return hipErrorInvalidValue; } @@ -2111,4 +2128,4 @@ hipError_t hipMallocHost(void** ptr, } HIP_RETURN(ihipMalloc(ptr, size, CL_MEM_SVM_FINE_GRAIN_BUFFER)); -} \ No newline at end of file +} From 9a0c5d0653defaa629987f10c0afbaf771f75624 Mon Sep 17 00:00:00 2001 From: Sameer Sahasrabuddhe <41661541+ssahasra@users.noreply.github.com> Date: Tue, 17 Mar 2020 14:03:27 +0530 Subject: [PATCH 142/154] enable HCC printf when using hip-clang This is cherry-picked from PR#1947 that was committed to the github repo. It allows printf to work with hip-clang and HCC runtime. Change-Id: I754753250ea1e694cf3441722e2d4c9d25fa75bc --- include/hip/hcc_detail/device_functions.h | 14 ++++++++++---- include/hip/hcc_detail/hip_runtime.h | 14 -------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/include/hip/hcc_detail/device_functions.h b/include/hip/hcc_detail/device_functions.h index 1be853bc99..e6549dde0d 100644 --- a/include/hip/hcc_detail/device_functions.h +++ b/include/hip/hcc_detail/device_functions.h @@ -34,13 +34,19 @@ THE SOFTWARE. #include #include -#if __HIP_CLANG_ONLY__ -#if __HIP_VDI__ +#if __HIP_CLANG_ONLY__ && __HIP_VDI__ extern "C" __device__ int printf(const char *fmt, ...); #else +#if HC_FEATURE_PRINTF +template +static inline __device__ void printf(const char* format, All... all) { + hc::printf(format, all...); +} +#else +template static inline __device__ void printf(const char* format, All... all) {} -#endif -#endif +#endif // HC_FEATURE_PRINTF +#endif // __HIP_CLANG_ONLY__ && __HIP_VDI__ /* Integer Intrinsics diff --git a/include/hip/hcc_detail/hip_runtime.h b/include/hip/hcc_detail/hip_runtime.h index cbb3a0e99a..ef591b8c51 100644 --- a/include/hip/hcc_detail/hip_runtime.h +++ b/include/hip/hcc_detail/hip_runtime.h @@ -314,20 +314,6 @@ extern "C" __device__ void* __hip_free(void* ptr); static inline __device__ void* malloc(size_t size) { return __hip_malloc(size); } static inline __device__ void* free(void* ptr) { return __hip_free(ptr); } -// Declare printf only for the HCC compiler. hip-clang is handled in -// device_functions.h -#if __HCC_ACCELERATOR__ -#if HC_FEATURE_PRINTF -template -static inline __device__ void printf(const char* format, All... all) { - hc::printf(format, all...); -} -#else -template -static inline __device__ void printf(const char* format, All... all) {} -#endif // HC_FEATURE_PRINTF -#endif // __HCC_ACCELERATOR__ - #endif //__HCC_OR_HIP_CLANG__ #ifdef __HCC__ From 3a690e960fcbfe0c096d8518b6c3d513955df972 Mon Sep 17 00:00:00 2001 From: Michael LIAO Date: Mon, 30 Mar 2020 09:10:16 -0400 Subject: [PATCH 143/154] [vdi] Fix calculation of MaxWaves - Consider the case where `usedVGPRs` is zero. - This fixes [SWDEV-228537](http://ontrack-internal.amd.com/browse/SWDEV-228537) Change-Id: I8675311f5fe24fb59c5d45bada122afefb55b128 --- vdi/hip_platform.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/vdi/hip_platform.cpp b/vdi/hip_platform.cpp index 688181870c..747f7457a0 100755 --- a/vdi/hip_platform.cpp +++ b/vdi/hip_platform.cpp @@ -700,17 +700,16 @@ hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, int* nu } // Find threads accupancy per CU => simd_per_cu * GPR usage constexpr size_t MaxWavesPerSimd = 8; // Limited by SPI 32 per CU, hence 8 per SIMD - size_t VgprWaves = wrkGrpInfo->availableVGPRs_ / amd::alignUp(wrkGrpInfo->usedVGPRs_, 4); - - size_t GprWaves; + size_t VgprWaves = MaxWavesPerSimd; + if (wrkGrpInfo->usedVGPRs_ > 0) { + VgprWaves = wrkGrpInfo->availableVGPRs_ / amd::alignUp(wrkGrpInfo->usedVGPRs_, 4); + } + size_t GprWaves = VgprWaves; if (wrkGrpInfo->usedSGPRs_ > 0) { const size_t maxSGPRs = (device->info().gfxipVersion_ < 800) ? 512 : 800; size_t SgprWaves = maxSGPRs / amd::alignUp(wrkGrpInfo->usedSGPRs_, 16); GprWaves = std::min(VgprWaves, SgprWaves); } - else { - GprWaves = VgprWaves; - } size_t alu_accupancy = device->info().simdPerCU_ * std::min(MaxWavesPerSimd, GprWaves); alu_accupancy *= wrkGrpInfo->wavefrontSize_; From f1566f6dc5378fb088bcef0084ff6c83540da830 Mon Sep 17 00:00:00 2001 From: Michael LIAO Date: Mon, 23 Mar 2020 14:28:46 -0400 Subject: [PATCH 144/154] [hipcc] Remove the previous workaround. Change-Id: Ie1c2aa21689bc343c10fd4b699546cf500ced5b2 --- bin/hipcc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bin/hipcc b/bin/hipcc index 013179b49e..0163324d65 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -470,13 +470,6 @@ foreach $arg (@ARGV) $optArg = $arg; } - ## This is a temporary workaround for CMake detection of OpenMP support. - ## It should be removed when the OpenMP detection c++ test in CMake is updated - ## and corrected CMake version is available. - if((defined $HIP_COMPILER) and ($HIP_COMPILER eq "clang") and ($arg eq '-fopenmp')) { - $HIPCXXFLAGS .= " -D_OPENMP " - } - ## process linker response file for hip-clang ## extract object files from static library and pass them directly to ## hip-clang in command line. From bdb3a4b393198bcb5052037c3f31c90ce521375f Mon Sep 17 00:00:00 2001 From: agodavar Date: Mon, 30 Mar 2020 11:29:47 -0400 Subject: [PATCH 145/154] Update Enable/Disable peers to match cuda behaviour Change-Id: I67194ccf77a0019368579ff7d95b7790fcf228f3 --- tests/src/p2p/hipPeerToPeer_simple.cpp | 2 +- vdi/hip_internal.hpp | 22 +++++++++++++++ vdi/hip_peer.cpp | 38 ++++++++++++++++---------- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/tests/src/p2p/hipPeerToPeer_simple.cpp b/tests/src/p2p/hipPeerToPeer_simple.cpp index c2e5965e39..90e7112356 100644 --- a/tests/src/p2p/hipPeerToPeer_simple.cpp +++ b/tests/src/p2p/hipPeerToPeer_simple.cpp @@ -23,7 +23,7 @@ THE SOFTWARE. // Also serves as a template for other tests. /* HIT_START - * BUILD: %t %s ../test_common.cpp EXCLUDE_HIP_PLATFORM vdi + * BUILD: %t %s ../test_common.cpp * TEST: %t EXCLUDE_HIP_PLATFORM hcc vdi * TEST: %t --memcpyWithPeer EXCLUDE_HIP_PLATFORM hcc vdi * TEST: %t --mirrorPeers EXCLUDE_HIP_PLATFORM hcc vdi diff --git a/vdi/hip_internal.hpp b/vdi/hip_internal.hpp index 129978dd68..a1cbe33a29 100755 --- a/vdi/hip_internal.hpp +++ b/vdi/hip_internal.hpp @@ -73,11 +73,14 @@ namespace hip { /// HIP Device class class Device { + amd::Monitor lock_{"Device lock"}; /// VDI context amd::Context* context_; /// Device's ID /// Store it here so we don't have to loop through the device list every time int deviceId_; + //Maintain list of user enabled peers + std::list userEnabledPeers; public: Device(amd::Context* ctx, int devId): context_(ctx), deviceId_(devId) { assert(ctx != nullptr); } ~Device() {} @@ -87,6 +90,25 @@ namespace hip { void retain() const { context_->retain(); } void release() const { context_->release(); } const std::vector& devices() const { return context_->devices(); } + hipError_t EnablePeerAccess(int peerDeviceId){ + amd::ScopedLock lock(lock_); + bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end()); + if (found) { + return hipErrorPeerAccessAlreadyEnabled; + } + userEnabledPeers.push_back(peerDeviceId); + return hipSuccess; + } + hipError_t DisablePeerAccess(int peerDeviceId) { + amd::ScopedLock lock(lock_); + bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end()); + if (found) { + userEnabledPeers.remove(peerDeviceId); + return hipSuccess; + } else { + return hipErrorPeerAccessNotEnabled; + } + } }; extern std::once_flag g_ihipInitialized; diff --git a/vdi/hip_peer.cpp b/vdi/hip_peer.cpp index 13cfd1cc7e..225361d525 100644 --- a/vdi/hip_peer.cpp +++ b/vdi/hip_peer.cpp @@ -48,48 +48,56 @@ hipError_t hipMemcpyPeerAsync(void* dst, hipCtx_t dstDevice, const void* src, hi HIP_RETURN(hipErrorNotSupported); } -hipError_t hipDeviceCanAccessPeer(int* canAccessPeer, int deviceId, int peerDeviceId) { - HIP_INIT_API(hipDeviceCanAccessPeer, canAccessPeer, deviceId, peerDeviceId); - +hipError_t canAccessPeer(int* canAccessPeer, int deviceId, int peerDeviceId){ amd::Device* device = nullptr; amd::Device* peer_device = nullptr; - if (canAccessPeer == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - /* Peer cannot be self */ if (deviceId == peerDeviceId) { *canAccessPeer = 0; - return HIP_RETURN(hipSuccess); + HIP_RETURN(hipSuccess); } - /* Cannot exceed the max number of devices */ if (static_cast(deviceId) >= g_devices.size() || static_cast(peerDeviceId) >= g_devices.size()) { - return HIP_RETURN(hipErrorInvalidValue); + HIP_RETURN(hipErrorInvalidDevice); } - device = g_devices[deviceId]->devices()[0]; peer_device = g_devices[peerDeviceId]->devices()[0]; - *canAccessPeer = static_cast(std::find(device->p2pDevices_.begin(), device->p2pDevices_.end(), as_cl(peer_device)) != device->p2pDevices_.end()); + HIP_RETURN(hipSuccess); +} - return HIP_RETURN(hipSuccess); +hipError_t hipDeviceCanAccessPeer(int* canAccess, int deviceId, int peerDeviceId) { + HIP_INIT_API(hipDeviceCanAccessPeer, canAccess, deviceId, peerDeviceId); + HIP_RETURN(canAccessPeer(canAccess, deviceId, peerDeviceId)); } hipError_t hipDeviceDisablePeerAccess(int peerDeviceId) { HIP_INIT_API(hipDeviceDisablePeerAccess, peerDeviceId); - - HIP_RETURN(hipSuccess); + int deviceId = hip::getCurrentDevice()->deviceId(); + int canAccess = 0; + if ((hipSuccess != canAccessPeer(&canAccess, deviceId, peerDeviceId)) || (canAccess == 0)) { + HIP_RETURN(hipErrorInvalidDevice); + } + HIP_RETURN(hip::getCurrentDevice()->DisablePeerAccess(peerDeviceId)); } hipError_t hipDeviceEnablePeerAccess(int peerDeviceId, unsigned int flags) { HIP_INIT_API(hipDeviceEnablePeerAccess, peerDeviceId, flags); - - HIP_RETURN(hipSuccess); + int deviceId = hip::getCurrentDevice()->deviceId(); + int canAccess = 0; + if (flags != 0) { + HIP_RETURN(hipErrorInvalidValue); + } + if ((hipSuccess != canAccessPeer(&canAccess, deviceId, peerDeviceId)) || (canAccess == 0)) { + HIP_RETURN(hipErrorInvalidDevice); + } + HIP_RETURN(hip::getCurrentDevice()->EnablePeerAccess(peerDeviceId)); } hipError_t hipMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, From b081912962e887f7f5b0875075d2f7e10c07e38b Mon Sep 17 00:00:00 2001 From: kjayapra-amd Date: Mon, 30 Mar 2020 11:32:27 -0400 Subject: [PATCH 146/154] SWDEV-216213 - Use different static & dynamic module maps for faster lookup. Change-Id: Ia605e76a411ad5be04046b9d61f1ac111d49bb4a --- vdi/hip_internal.hpp | 11 ++++++++- vdi/hip_module.cpp | 16 ++---------- vdi/hip_platform.cpp | 58 +++++++++++++++++++++++++++++--------------- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/vdi/hip_internal.hpp b/vdi/hip_internal.hpp index a1cbe33a29..d5d35fcf1d 100755 --- a/vdi/hip_internal.hpp +++ b/vdi/hip_internal.hpp @@ -216,7 +216,6 @@ public: std::string deviceName; std::vector< std::pair< hipModule_t, bool > >* modules; std::vector functions; - bool dyn_mod; }; struct DeviceVar { void* shadowVptr; @@ -227,6 +226,15 @@ public: bool dyn_undef; }; private: + class Module { + public: + Module(hipModule_t hip_module_) : hip_module(hip_module_) {} + std::unordered_map functions_; + private: + hipModule_t hip_module; + }; + std::unordered_map module_map_; + std::unordered_map functions_; std::unordered_multimap vars_; @@ -247,6 +255,7 @@ public: void registerVar(const void* hostvar, const DeviceVar& var); void registerFunction(const void* hostFunction, const DeviceFunction& func); + bool registerModFuncs(std::vector& func_names, hipModule_t* module); bool findModFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name); bool createFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name); hipFunction_t getFunc(const void* hostFunction, int deviceId); diff --git a/vdi/hip_module.cpp b/vdi/hip_module.cpp index 0e22f9063b..a7822b315e 100755 --- a/vdi/hip_module.cpp +++ b/vdi/hip_module.cpp @@ -160,27 +160,15 @@ inline bool ihipModuleRegisterFunc(amd::Program* program, hipModule_t* module) { device::Program* dev_program = program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]); - // Get all the global func names from COMGR if (!dev_program->getGlobalFuncFromCodeObj(&func_names)) { return false; } - for (auto it = func_names.begin(); it != func_names.end(); ++it) { - auto modules = new std::vector >(g_devices.size()); - for (size_t dev = 0; dev < g_devices.size(); ++dev) { - modules->at(dev) = std::make_pair(*module, true); - } - - // Create a new pointer, since the hostFunction* wont be available - // if device code not in the same file as host code. - PlatformState::DeviceFunction dfunc{*it, modules, std::vector{ g_devices.size() }, true}; - PlatformState::instance().registerFunction(new std::string(it->c_str()), dfunc); - } - - return true; + return PlatformState::instance().registerModFuncs(func_names, module); } + inline bool ihipModuleRegisterGlobal(amd::Program* program, hipModule_t* module) { size_t var_size = 0; diff --git a/vdi/hip_platform.cpp b/vdi/hip_platform.cpp index 747f7457a0..0ddc2fe8af 100755 --- a/vdi/hip_platform.cpp +++ b/vdi/hip_platform.cpp @@ -190,23 +190,21 @@ void PlatformState::init() bool PlatformState::unregisterFunc(hipModule_t hmod) { amd::ScopedLock lock(lock_); - auto it = functions_.begin(); - while (it != functions_.end()) { - DeviceFunction& dfunc = it->second; - if ((*dfunc.modules)[0].first == hmod) { - if (dfunc.dyn_mod) { - std::string *s = reinterpret_cast(const_cast(it->first)); - delete s; - } - for (size_t dev = 0; dev < g_devices.size(); ++dev) { - if (dfunc.functions[dev] != 0) { - hip::Function* f = reinterpret_cast(dfunc.functions[dev]); - delete f; + auto mod_it = module_map_.find(hmod); + if (mod_it != module_map_.cend()) { + PlatformState::Module* mod_ptr = mod_it->second; + if(mod_ptr != nullptr) { + for (auto func_it = mod_ptr->functions_.begin(); func_it != mod_ptr->functions_.end(); ++func_it) { + PlatformState::DeviceFunction &devFunc = func_it->second; + for (size_t dev = 0; dev < g_devices.size(); ++dev) { + if (devFunc.functions[dev] != 0) { + hip::Function* f = reinterpret_cast(devFunc.functions[dev]); + delete f; + } } + delete devFunc.modules; } - functions_.erase(it++); - } else { - ++it; + delete mod_ptr; } } return true; @@ -316,11 +314,33 @@ bool CL_CALLBACK getSvarInfo(cl_program program, std::string var_name, void** va var_addr, var_size); } +bool PlatformState::registerModFuncs(std::vector& func_names, hipModule_t* module) { + amd::ScopedLock lock(lock_); + PlatformState::Module* mod_ptr = new PlatformState::Module(*module); + + for (auto it = func_names.begin(); it != func_names.end(); ++it) { + auto modules = new std::vector >(g_devices.size()); + for (size_t dev = 0; dev < g_devices.size(); ++dev) { + modules->at(dev) = std::make_pair(*module, true); + } + + PlatformState::DeviceFunction dfunc{*it, modules, + std::vector(g_devices.size(), 0)}; + mod_ptr->functions_.insert(std::make_pair(*it, dfunc)); + } + + module_map_.insert(std::make_pair(*module, mod_ptr)); + return true; +} + bool PlatformState::findModFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name) { amd::ScopedLock lock(lock_); - for (auto it = functions_.begin(); it != functions_.end(); ++it) { - PlatformState::DeviceFunction& devFunc = it->second; - if ((devFunc.deviceName == name) && (hmod == (*devFunc.modules)[ihipGetDevice()].first)) { + + auto mod_it = module_map_.find(hmod); + if (mod_it != module_map_.cend()) { + auto func_it = mod_it->second->functions_.find(name); + if (func_it != mod_it->second->functions_.cend()) { + PlatformState::DeviceFunction& devFunc = func_it->second; if (devFunc.functions[ihipGetDevice()] == 0) { if(!createFunc(&devFunc.functions[ihipGetDevice()], hmod, name)) { return false; @@ -496,7 +516,7 @@ extern "C" void __hipRegisterFunction( dim3* gridDim, int* wSize) { - PlatformState::DeviceFunction func{ std::string{deviceName}, modules, std::vector{g_devices.size()}, false}; + PlatformState::DeviceFunction func{ std::string{deviceName}, modules, std::vector{g_devices.size()}}; PlatformState::instance().registerFunction(hostFunction, func); // for (size_t i = 0; i < g_devices.size(); ++i) { // PlatformState::instance().getFunc(hostFunction, i); From 94a7ef6ed1dd569214dc62895e7b958eade83d0a Mon Sep 17 00:00:00 2001 From: Christophe Paquot Date: Mon, 30 Mar 2020 12:37:08 -0700 Subject: [PATCH 147/154] Do not retry to allocate when OOM. Shouldn't be needed since we idle on Free. SWDEV-229214 Change-Id: I183006f409388e3c7981f2569649d01d6378be46 --- vdi/hip_memory.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 3fc1a2921a..083b855fa9 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -64,16 +64,7 @@ hipError_t ihipMalloc(void** ptr, size_t sizeBytes, unsigned int flags) *ptr = amd::SvmBuffer::malloc(*amdContext, flags, sizeBytes, amdContext->devices()[0]->info().memBaseAddrAlign_); if (*ptr == nullptr) { - - for (auto& dev : g_devices) { - hip::getNullStream(*dev->asContext())->finish(); - hip::syncStreams(dev->deviceId()); - } - - *ptr = amd::SvmBuffer::malloc(*amdContext, flags, sizeBytes, amdContext->devices()[0]->info().memBaseAddrAlign_); - if (*ptr == nullptr) { - return hipErrorOutOfMemory; - } + return hipErrorOutOfMemory; } ClPrint(amd::LOG_INFO, amd::LOG_API, "ihipMalloc ptr=0x%zx", *ptr); return hipSuccess; From b72196613a60ceacb0c167c204739194260a9893 Mon Sep 17 00:00:00 2001 From: Michael LIAO Date: Wed, 4 Mar 2020 16:05:37 -0500 Subject: [PATCH 148/154] [vdi] Fix hipGetSymbol{Address|Size} - Use symbol value as the qeury key. Compared to the symbol name, the symbol value is more robust as developers may use unqualified or qualified identifiers. It also removes the mangling and/or demangling requirement for the runtime API. Change-Id: I9d4259f3842612c7cc98551269fc2092d8b5c19e --- include/hip/hcc_detail/hip_runtime.h | 2 +- include/hip/hcc_detail/hip_runtime_api.h | 52 +++++++++++++++++++++--- vdi/hip_internal.hpp | 8 +++- vdi/hip_memory.cpp | 40 ++++++++++++------ vdi/hip_platform.cpp | 39 +++++++++++++++--- 5 files changed, 114 insertions(+), 27 deletions(-) diff --git a/include/hip/hcc_detail/hip_runtime.h b/include/hip/hcc_detail/hip_runtime.h index ef591b8c51..a3a0963ba9 100644 --- a/include/hip/hcc_detail/hip_runtime.h +++ b/include/hip/hcc_detail/hip_runtime.h @@ -379,7 +379,7 @@ extern void ihipPostLaunchKernel(const char* kernelName, hipStream_t stream, gri #elif defined(__clang__) && defined(__HIP__) #define HIP_KERNEL_NAME(...) __VA_ARGS__ -#define HIP_SYMBOL(X) #X +#define HIP_SYMBOL(X) X typedef int hipLaunchParm; diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index d35d426a5e..2b5649e721 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -1488,18 +1488,18 @@ hipError_t hipMemcpyDtoDAsync(hipDeviceptr_t dst, hipDeviceptr_t src, size_t siz hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes, hipModule_t hmod, const char* name); -hipError_t hipGetSymbolAddress(void** devPtr, const void* symbolName); -hipError_t hipGetSymbolSize(size_t* size, const void* symbolName); -hipError_t hipMemcpyToSymbol(const void* symbolName, const void* src, +hipError_t hipGetSymbolAddress(void** devPtr, const void* symbol); +hipError_t hipGetSymbolSize(size_t* size, const void* symbol); +hipError_t hipMemcpyToSymbol(const void* symbol, const void* src, size_t sizeBytes, size_t offset __dparm(0), hipMemcpyKind kind __dparm(hipMemcpyHostToDevice)); -hipError_t hipMemcpyToSymbolAsync(const void* symbolName, const void* src, +hipError_t hipMemcpyToSymbolAsync(const void* symbol, const void* src, size_t sizeBytes, size_t offset, hipMemcpyKind kind, hipStream_t stream __dparm(0)); -hipError_t hipMemcpyFromSymbol(void* dst, const void* symbolName, +hipError_t hipMemcpyFromSymbol(void* dst, const void* symbol, size_t sizeBytes, size_t offset __dparm(0), hipMemcpyKind kind __dparm(hipMemcpyDeviceToHost)); -hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* symbolName, +hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* symbol, size_t sizeBytes, size_t offset, hipMemcpyKind kind, hipStream_t stream __dparm(0)); @@ -3517,6 +3517,46 @@ static hipError_t __host__ inline hipOccupancyMaxActiveBlocksPerMultiprocessorWi } #endif // defined(__cplusplus) && !defined(__HCC__) && defined(__clang__) && defined(__HIP__) +#if defined(__cplusplus) && !defined(__HCC__) + +template +hipError_t hipGetSymbolAddress(void** devPtr, const T &symbol) { + return ::hipGetSymbolAddress(devPtr, (const void *)&symbol); +} + +template +hipError_t hipGetSymbolSize(size_t* size, const T &symbol) { + return ::hipGetSymbolSize(size, (const void *)&symbol); +} + +template +hipError_t hipMemcpyToSymbol(const T& symbol, const void* src, size_t sizeBytes, + size_t offset __dparm(0), + hipMemcpyKind kind __dparm(hipMemcpyHostToDevice)) { + return ::hipMemcpyToSymbol((const void*)&symbol, src, sizeBytes, offset, kind); +} + +template +hipError_t hipMemcpyToSymbolAsync(const T& symbol, const void* src, size_t sizeBytes, size_t offset, + hipMemcpyKind kind, hipStream_t stream __dparm(0)) { + return ::hipMemcpyToSymbolAsync((const void*)&symbol, src, sizeBytes, offset, kind, stream); +} + +template +hipError_t hipMemcpyFromSymbol(void* dst, const T &symbol, + size_t sizeBytes, size_t offset __dparm(0), + hipMemcpyKind kind __dparm(hipMemcpyDeviceToHost)) { + return ::hipMemcpyFromSymbol(dst, (const void*)&symbol, sizeBytes, offset, kind); +} + +template +hipError_t hipMemcpyFromSymbolAsync(void* dst, const T& symbol, size_t sizeBytes, size_t offset, + hipMemcpyKind kind, hipStream_t stream __dparm(0)) { + return ::hipMemcpyFromSymbolAsync(dst, (const void*)&symbol, sizeBytes, offset, kind, stream); +} + +#endif + #if USE_PROF_API #include #endif diff --git a/vdi/hip_internal.hpp b/vdi/hip_internal.hpp index d5d35fcf1d..92397d1255 100755 --- a/vdi/hip_internal.hpp +++ b/vdi/hip_internal.hpp @@ -237,6 +237,8 @@ private: std::unordered_map functions_; std::unordered_multimap vars_; + // Map from the host shadow symbol to its device name. + std::unordered_map symbols_; static PlatformState* platform_; @@ -251,8 +253,10 @@ public: std::vector< std::pair >* unregisterVar(hipModule_t hmod); + bool findSymbol(const void *hostVar, std::string &devName); PlatformState::DeviceVar* findVar(std::string hostVar, int deviceId, hipModule_t hmod); - void registerVar(const void* hostvar, const DeviceVar& var); + void registerVarSym(const void *hostVar, const char *symbolName); + void registerVar(const char* symbolName, const DeviceVar& var); void registerFunction(const void* hostFunction, const DeviceFunction& func); bool registerModFuncs(std::vector& func_names, hipModule_t* module); @@ -260,7 +264,7 @@ public: bool createFunc(hipFunction_t* hfunc, hipModule_t hmod, const char* name); hipFunction_t getFunc(const void* hostFunction, int deviceId); bool getFuncAttr(const void* hostFunction, hipFuncAttributes* func_attr); - bool getGlobalVar(const void* hostVar, int deviceId, hipModule_t hmod, + bool getGlobalVar(const char* hostVar, int deviceId, hipModule_t hmod, hipDeviceptr_t* dev_ptr, size_t* size_ptr); bool getTexRef(const char* hostVar, hipModule_t hmod, textureReference** texRef); diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 083b855fa9..78288a6a78 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -717,15 +717,19 @@ hipError_t hipHostAlloc(void** ptr, size_t sizeBytes, unsigned int flags) { }; -hipError_t hipMemcpyToSymbol(const void* symbolName, const void* src, size_t count, +hipError_t hipMemcpyToSymbol(const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyToSymbol, symbolName, src, count, offset, kind); + HIP_INIT_API(hipMemcpyToSymbol, symbol, src, count, offset, kind); size_t sym_size = 0; hipDeviceptr_t device_ptr = nullptr; + std::string symbolName; + if (!PlatformState::instance().findSymbol(symbol, symbolName)) { + HIP_RETURN(hipErrorInvalidSymbol); + } /* Get address and size for the global symbol */ - if (!PlatformState::instance().getGlobalVar(symbolName, ihipGetDevice(), nullptr, + if (!PlatformState::instance().getGlobalVar(symbolName.c_str(), ihipGetDevice(), nullptr, &device_ptr, &sym_size)) { HIP_RETURN(hipErrorInvalidSymbol); } @@ -741,15 +745,19 @@ hipError_t hipMemcpyToSymbol(const void* symbolName, const void* src, size_t cou HIP_RETURN(hipMemcpy(device_ptr, src, count, kind)); } -hipError_t hipMemcpyFromSymbol(void* dst, const void* symbolName, size_t count, +hipError_t hipMemcpyFromSymbol(void* dst, const void* symbol, size_t count, size_t offset, hipMemcpyKind kind) { - HIP_INIT_API(hipMemcpyFromSymbol, symbolName, dst, count, offset, kind); + HIP_INIT_API(hipMemcpyFromSymbol, symbol, dst, count, offset, kind); size_t sym_size = 0; hipDeviceptr_t device_ptr = nullptr; + std::string symbolName; + if (!PlatformState::instance().findSymbol(symbol, symbolName)) { + HIP_RETURN(hipErrorInvalidSymbol); + } /* Get address and size for the global symbol */ - if (!PlatformState::instance().getGlobalVar(symbolName, ihipGetDevice(), nullptr, + if (!PlatformState::instance().getGlobalVar(symbolName.c_str(), ihipGetDevice(), nullptr, &device_ptr, &sym_size)) { HIP_RETURN(hipErrorInvalidSymbol); } @@ -765,15 +773,19 @@ hipError_t hipMemcpyFromSymbol(void* dst, const void* symbolName, size_t count, HIP_RETURN(hipMemcpy(dst, device_ptr, count, kind)); } -hipError_t hipMemcpyToSymbolAsync(const void* symbolName, const void* src, size_t count, +hipError_t hipMemcpyToSymbolAsync(const void* symbol, const void* src, size_t count, size_t offset, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpyToSymbolAsync, symbolName, src, count, offset, kind, stream); + HIP_INIT_API(hipMemcpyToSymbolAsync, symbol, src, count, offset, kind, stream); size_t sym_size = 0; hipDeviceptr_t device_ptr = nullptr; + std::string symbolName; + if (!PlatformState::instance().findSymbol(symbol, symbolName)) { + HIP_RETURN(hipErrorInvalidSymbol); + } /* Get address and size for the global symbol */ - if (!PlatformState::instance().getGlobalVar(symbolName, ihipGetDevice(), nullptr, + if (!PlatformState::instance().getGlobalVar(symbolName.c_str(), ihipGetDevice(), nullptr, &device_ptr, &sym_size)) { HIP_RETURN(hipErrorInvalidSymbol); } @@ -789,15 +801,19 @@ hipError_t hipMemcpyToSymbolAsync(const void* symbolName, const void* src, size_ HIP_RETURN(hipMemcpyAsync(device_ptr, src, count, kind, stream)); } -hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* symbolName, size_t count, +hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* symbol, size_t count, size_t offset, hipMemcpyKind kind, hipStream_t stream) { - HIP_INIT_API(hipMemcpyFromSymbolAsync, symbolName, dst, count, offset, kind, stream); + HIP_INIT_API(hipMemcpyFromSymbolAsync, symbol, dst, count, offset, kind, stream); size_t sym_size = 0; hipDeviceptr_t device_ptr = nullptr; + std::string symbolName; + if (!PlatformState::instance().findSymbol(symbol, symbolName)) { + HIP_RETURN(hipErrorInvalidSymbol); + } /* Get address and size for the global symbol */ - if (!PlatformState::instance().getGlobalVar(symbolName, ihipGetDevice(), nullptr, + if (!PlatformState::instance().getGlobalVar(symbolName.c_str(), ihipGetDevice(), nullptr, &device_ptr, &sym_size)) { HIP_RETURN(hipErrorInvalidSymbol); } diff --git a/vdi/hip_platform.cpp b/vdi/hip_platform.cpp index 0ddc2fe8af..ca79c5300d 100755 --- a/vdi/hip_platform.cpp +++ b/vdi/hip_platform.cpp @@ -266,7 +266,21 @@ PlatformState::DeviceVar* PlatformState::findVar(std::string hostVar, int device return dvar; } -void PlatformState::registerVar(const void* hostvar, +bool PlatformState::findSymbol(const void *hostVar, std::string &symbolName) { + auto it = symbols_.find(hostVar); + if (it != symbols_.end()) { + symbolName = it->second; + return true; + } + return false; +} + +void PlatformState::registerVarSym(const void *hostVar, const char *symbolName) { + amd::ScopedLock lock(lock_); + symbols_.insert(std::make_pair(hostVar, std::string(symbolName))); +} + +void PlatformState::registerVar(const char* hostvar, const DeviceVar& rvar) { amd::ScopedLock lock(lock_); vars_.insert(std::make_pair(std::string(reinterpret_cast(hostvar)), rvar)); @@ -447,7 +461,7 @@ bool PlatformState::getTexRef(const char* hostVar, hipModule_t hmod, textureRefe return true; } -bool PlatformState::getGlobalVar(const void* hostVar, int deviceId, hipModule_t hmod, +bool PlatformState::getGlobalVar(const char* hostVar, int deviceId, hipModule_t hmod, hipDeviceptr_t* dev_ptr, size_t* size_ptr) { amd::ScopedLock lock(lock_); DeviceVar* dvar = findVar(std::string(reinterpret_cast(hostVar)), deviceId, hmod); @@ -542,6 +556,7 @@ extern "C" void __hipRegisterVar( std::vector{g_devices.size()}, false }; PlatformState::instance().registerVar(hostVar, dvar); + PlatformState::instance().registerVarSym(var, deviceVar); } extern "C" void __hipUnregisterFatBinary(std::vector< std::pair >* modules) @@ -643,18 +658,30 @@ extern "C" hipError_t hipLaunchByPtr(const void *hostFunction) exec.sharedMem_, exec.hStream_, nullptr, extra)); } -hipError_t hipGetSymbolAddress(void** devPtr, const void* symbolName) { +hipError_t hipGetSymbolAddress(void** devPtr, const void* symbol) { + HIP_INIT_API(hipGetSymbolAddress, devPtr, symbol); + + std::string symbolName; + if (!PlatformState::instance().findSymbol(symbol, symbolName)) { + HIP_RETURN(hipErrorInvalidSymbol); + } size_t size = 0; - if(!PlatformState::instance().getGlobalVar(symbolName, ihipGetDevice(), nullptr, + if(!PlatformState::instance().getGlobalVar(symbolName.c_str(), ihipGetDevice(), nullptr, devPtr, &size)) { HIP_RETURN(hipErrorInvalidSymbol); } HIP_RETURN(hipSuccess); } -hipError_t hipGetSymbolSize(size_t* sizePtr, const void* symbolName) { +hipError_t hipGetSymbolSize(size_t* sizePtr, const void* symbol) { + HIP_INIT_API(hipGetSymbolSize, sizePtr, symbol); + + std::string symbolName; + if (!PlatformState::instance().findSymbol(symbol, symbolName)) { + HIP_RETURN(hipErrorInvalidSymbol); + } hipDeviceptr_t devPtr = nullptr; - if (!PlatformState::instance().getGlobalVar(symbolName, ihipGetDevice(), nullptr, + if (!PlatformState::instance().getGlobalVar(symbolName.c_str(), ihipGetDevice(), nullptr, &devPtr, sizePtr)) { HIP_RETURN(hipErrorInvalidSymbol); } From e4ea1f60bbe0500e6334c59d8a383403e0327c5f Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Thu, 26 Mar 2020 15:07:41 -0700 Subject: [PATCH 149/154] Cleanup stream from hip:Event class. Change-Id: I98de07d33bb7fea8f5e2d32b288c15f10ce58902 --- vdi/hip_event.cpp | 16 +++------------- vdi/hip_event.hpp | 5 ++++- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/vdi/hip_event.cpp b/vdi/hip_event.cpp index 03498c47cf..677becd67e 100644 --- a/vdi/hip_event.cpp +++ b/vdi/hip_event.cpp @@ -95,16 +95,13 @@ hipError_t Event::elapsedTime(Event& eStop, float& ms) { } hipError_t Event::streamWait(amd::HostQueue* hostQueue, uint flags) { - if (stream_ == hostQueue) return hipSuccess; + if ((event_ == nullptr) || (event_->command().queue() == hostQueue)) { + return hipSuccess; + } amd::ScopedLock lock(lock_); bool retain = false; - if (event_ == nullptr) { - event_ = stream_->getLastQueuedCommand(true); - retain = true; - } - if (!event_->notifyCmdQueue()) { return hipErrorLaunchOutOfResources; } @@ -118,19 +115,12 @@ hipError_t Event::streamWait(amd::HostQueue* hostQueue, uint flags) { command->enqueue(); command->release(); - if (retain) { - event_->release(); - event_ = nullptr; - } - return hipSuccess; } void Event::addMarker(amd::HostQueue* queue, amd::Command* command) { amd::ScopedLock lock(lock_); - stream_ = queue; - if (event_ == &command->event()) return; if (event_ != nullptr) { diff --git a/vdi/hip_event.hpp b/vdi/hip_event.hpp index a9cae92bbb..2360c972bb 100644 --- a/vdi/hip_event.hpp +++ b/vdi/hip_event.hpp @@ -37,7 +37,10 @@ public: class Event { public: - Event(unsigned int flags) : flags(flags), lock_("hipEvent_t"), stream_(getNullStream()), event_(nullptr) {} + Event(unsigned int flags) : flags(flags), lock_("hipEvent_t"), event_(nullptr) { + // No need to init event_ here as addMarker does that + } + ~Event() { if (event_ != nullptr) { event_->release(); From bb62f8553fdfd5d14caa0e2bac1c4a21ef89fb62 Mon Sep 17 00:00:00 2001 From: Reshabh Sharma <60782296+rksharma-yymd@users.noreply.github.com> Date: Tue, 17 Mar 2020 14:02:14 +0530 Subject: [PATCH 150/154] Output file name should not change flags picked for compiler (#1938) Fixes SWDEV-207362, The output file name should not contribute to picking up the right flags for the compiler. This fix solves issues when the output has conflicting extensions which confuses hipcc to treat them as the source files and add the required flags for them. PS: Output file refers to the file followed by -o Change-Id: I1095966c11143ad73e81fabc35b4e9de5d3afada Example: hipcc test.o -o test.hip will add the flags for .hip compilation ignoring the fact that it is an output file --- bin/hipcc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bin/hipcc b/bin/hipcc index 0163324d65..b3db312c78 100755 --- a/bin/hipcc +++ b/bin/hipcc @@ -391,6 +391,7 @@ my $toolArgs = ""; # arguments to pass to the hcc or nvcc tool my $optArg = ""; # -O args my $targetOpt = '--amdgpu-target='; my $targetsStr = ""; +my $skipOutputFile = 0; # file followed by -o should not contibute in picking compiler flags foreach $arg (@ARGV) { @@ -402,8 +403,16 @@ foreach $arg (@ARGV) $needCXXFLAGS = 1; $needLDFLAGS = 0; } + + if ($skipOutputFile) { + $toolArgs .= " $arg"; + $skipOutputFile = 0; + next; + } + if ($arg eq '-o') { $needLDFLAGS = 1; + $skipOutputFile = 1; } if(($trimarg eq '-stdlib=libc++') and ($setStdLib eq 0)) From 77223a8ecaff71a518531f5cd7fb3313190651d2 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Tue, 31 Mar 2020 16:15:57 -0400 Subject: [PATCH 151/154] (SWDEV-229354) This patch is a workaround to support user pitch for hipMemcpy{2D/3D}. Historically OpenCL didn't support pitch with clEnqueueFillBuffer(), so neither did we in VDI. Adding it now will be slightly nontrivial, since the fill kernel and runtime in many places will need to be modified. As a temporary workaround for cases when pitch > width, we can just enqueue a fill for each row separately. This implementation is slow, but it satisfies the correctness criteria. Change-Id: Idfeca349288b51d6ff84a7cf001fb63c6a66818a --- vdi/hip_memory.cpp | 76 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/vdi/hip_memory.cpp b/vdi/hip_memory.cpp index 78288a6a78..5438064e9b 100644 --- a/vdi/hip_memory.cpp +++ b/vdi/hip_memory.cpp @@ -1734,35 +1734,91 @@ hipError_t hipMemsetD32Async(hipDeviceptr_t dst, int value, size_t count, HIP_RETURN(ihipMemset(dst, value, sizeof(int32_t), count * sizeof(int32_t), stream, true)); } +hipError_t ihipMemset3D(hipPitchedPtr pitchedDevPtr, + int value, + hipExtent extent, + hipStream_t stream, + bool isAsync = false) { + if (pitchedDevPtr.pitch == extent.width) { + return ihipMemset(pitchedDevPtr.ptr, value, sizeof(int8_t), extent.width * extent.height * extent.depth, stream, isAsync); + } + + // Workaround for cases when pitch > row untill fill kernel will be updated to support pitch. + // Fallback to filling one row at a time. + + amd::HostQueue* queue = hip::getQueue(stream); + + size_t offset = 0; + amd::Memory* memory = getMemoryObject(pitchedDevPtr.ptr, offset); + + amd::Coord3D origin(offset); + amd::Coord3D region(pitchedDevPtr.xsize, pitchedDevPtr.ysize, extent.depth); + amd::BufferRect rect; + if (!rect.create(static_cast(origin), static_cast(region), pitchedDevPtr.pitch, 0)) { + return hipErrorInvalidValue; + } + + if (memory != nullptr) { + std::vector commands; + + for (size_t slice = 0; slice < extent.depth; slice++) { + for (size_t row = 0; row < extent.height; row++) { + const size_t rowOffset = rect.offset(0, row, slice); + amd::FillMemoryCommand* command = new amd::FillMemoryCommand(*queue, + CL_COMMAND_FILL_BUFFER, + amd::Command::EventWaitList{}, + *memory->asBuffer(), + &value, + sizeof(int8_t), + amd::Coord3D{rowOffset, 0, 0}, + amd::Coord3D{extent.width, 1, 1}); + + command->enqueue(); + commands.push_back(command); + } + } + + for (auto &command: commands) { + if (!isAsync) { + command->awaitCompletion(); + } + command->release(); + } + } else { + for (size_t slice = 0; slice < extent.depth; slice++) { + for (size_t row = 0; row < extent.height; row++) { + const size_t rowOffset = rect.offset(0, row, slice); + std::memset(pitchedDevPtr.ptr, value, extent.width); + } + } + } + + return hipSuccess; +} + hipError_t hipMemset2D(void* dst, size_t pitch, int value, size_t width, size_t height) { HIP_INIT_API(hipMemset2D, dst, pitch, value, width, height); - HIP_RETURN(ihipMemset(dst, value, sizeof(int8_t), pitch * height, nullptr)); + HIP_RETURN(ihipMemset3D({dst, pitch, width, height}, value, {width, height, 1}, nullptr)); } hipError_t hipMemset2DAsync(void* dst, size_t pitch, int value, size_t width, size_t height, hipStream_t stream) { HIP_INIT_API(hipMemset2DAsync, dst, pitch, value, width, height, stream); - HIP_RETURN(ihipMemset(dst, value, sizeof(int8_t), pitch * height, stream, true)); + HIP_RETURN(ihipMemset3D({dst, pitch, width, height}, value, {width, height, 1}, stream, true)); } hipError_t hipMemset3D(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent) { HIP_INIT_API(hipMemset3D, pitchedDevPtr, value, extent); - void *dst = pitchedDevPtr.ptr; - size_t sizeBytes = pitchedDevPtr.pitch * extent.height * extent.depth; - - HIP_RETURN(ihipMemset(dst, value, sizeof(int8_t), sizeBytes, nullptr)); + HIP_RETURN(ihipMemset3D(pitchedDevPtr, value, extent, nullptr)); } hipError_t hipMemset3DAsync(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent, hipStream_t stream) { HIP_INIT_API(hipMemset3DAsync, pitchedDevPtr, value, extent, stream); - void *dst = pitchedDevPtr.ptr; - size_t sizeBytes = pitchedDevPtr.pitch * extent.height * extent.depth; - - HIP_RETURN(ihipMemset(dst, value, sizeof(int8_t), sizeBytes, stream, true)); + HIP_RETURN(ihipMemset3D(pitchedDevPtr, value, extent, stream, false)); } hipError_t hipMemAllocPitch(hipDeviceptr_t* dptr, size_t* pitch, size_t widthInBytes, From aea688b79c21027cd71a3d0080bfd6593862f2b1 Mon Sep 17 00:00:00 2001 From: Vladislav Sytchenko Date: Wed, 25 Mar 2020 15:32:42 -0400 Subject: [PATCH 152/154] Add entry points for hipTexObject*() API Even though the runtime and driver texture object API is one to one, the structs used by these APIs are not. See hipResourceDesc vs HIP_RESOURCE_DESC differences. These differences are not trivial and most likely won't be able to handled by hipify, so we need new API entry points. Change-Id: Id4bcb1ad0ae15378dbdb5a2ed07e5ea30f320082 --- include/hip/hcc_detail/driver_types.h | 129 ++++++++++++ include/hip/hcc_detail/hip_runtime_api.h | 21 ++ vdi/hip_conversions.hpp | 241 +++++++++++++++++++++++ vdi/hip_hcc.def.in | 5 + vdi/hip_hcc.map.in | 5 + vdi/hip_texture.cpp | 72 ++++++- 6 files changed, 472 insertions(+), 1 deletion(-) diff --git a/include/hip/hcc_detail/driver_types.h b/include/hip/hcc_detail/driver_types.h index d24f097f84..ae8e8b1757 100644 --- a/include/hip/hcc_detail/driver_types.h +++ b/include/hip/hcc_detail/driver_types.h @@ -135,6 +135,47 @@ typedef enum hipResourceType { hipResourceTypePitch2D = 0x03 }hipResourceType; +typedef enum HIPresourcetype_enum { + HIP_RESOURCE_TYPE_ARRAY = 0x00, /**< Array resoure */ + HIP_RESOURCE_TYPE_MIPMAPPED_ARRAY = 0x01, /**< Mipmapped array resource */ + HIP_RESOURCE_TYPE_LINEAR = 0x02, /**< Linear resource */ + HIP_RESOURCE_TYPE_PITCH2D = 0x03 /**< Pitch 2D resource */ +} HIPresourcetype; + +/** + * hip address modes + */ +typedef enum HIPaddress_mode_enum { + HIP_TR_ADDRESS_MODE_WRAP = 0, + HIP_TR_ADDRESS_MODE_CLAMP = 1, + HIP_TR_ADDRESS_MODE_MIRROR = 2, + HIP_TR_ADDRESS_MODE_BORDER = 3 +} HIPaddress_mode; + +/** + * hip filter modes + */ +typedef enum HIPfilter_mode_enum { + HIP_TR_FILTER_MODE_POINT = 0, + HIP_TR_FILTER_MODE_LINEAR = 1 +} HIPfilter_mode; + +/** + * Texture descriptor + */ +typedef struct HIP_TEXTURE_DESC_st { + HIPaddress_mode addressMode[3]; /**< Address modes */ + HIPfilter_mode filterMode; /**< Filter mode */ + unsigned int flags; /**< Flags */ + unsigned int maxAnisotropy; /**< Maximum anisotropy ratio */ + HIPfilter_mode mipmapFilterMode; /**< Mipmap filter mode */ + float mipmapLevelBias; /**< Mipmap level bias */ + float minMipmapLevelClamp; /**< Mipmap minimum level clamp */ + float maxMipmapLevelClamp; /**< Mipmap maximum level clamp */ + float borderColor[4]; /**< Border Color */ + int reserved[12]; +} HIP_TEXTURE_DESC; + /** * hip texture resource view formats */ @@ -176,6 +217,45 @@ typedef enum hipResourceViewFormat { hipResViewFormatUnsignedBlockCompressed7 = 0x22 }hipResourceViewFormat; +typedef enum HIPresourceViewFormat_enum +{ + HIP_RES_VIEW_FORMAT_NONE = 0x00, /**< No resource view format (use underlying resource format) */ + HIP_RES_VIEW_FORMAT_UINT_1X8 = 0x01, /**< 1 channel unsigned 8-bit integers */ + HIP_RES_VIEW_FORMAT_UINT_2X8 = 0x02, /**< 2 channel unsigned 8-bit integers */ + HIP_RES_VIEW_FORMAT_UINT_4X8 = 0x03, /**< 4 channel unsigned 8-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_1X8 = 0x04, /**< 1 channel signed 8-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_2X8 = 0x05, /**< 2 channel signed 8-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_4X8 = 0x06, /**< 4 channel signed 8-bit integers */ + HIP_RES_VIEW_FORMAT_UINT_1X16 = 0x07, /**< 1 channel unsigned 16-bit integers */ + HIP_RES_VIEW_FORMAT_UINT_2X16 = 0x08, /**< 2 channel unsigned 16-bit integers */ + HIP_RES_VIEW_FORMAT_UINT_4X16 = 0x09, /**< 4 channel unsigned 16-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_1X16 = 0x0a, /**< 1 channel signed 16-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_2X16 = 0x0b, /**< 2 channel signed 16-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_4X16 = 0x0c, /**< 4 channel signed 16-bit integers */ + HIP_RES_VIEW_FORMAT_UINT_1X32 = 0x0d, /**< 1 channel unsigned 32-bit integers */ + HIP_RES_VIEW_FORMAT_UINT_2X32 = 0x0e, /**< 2 channel unsigned 32-bit integers */ + HIP_RES_VIEW_FORMAT_UINT_4X32 = 0x0f, /**< 4 channel unsigned 32-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_1X32 = 0x10, /**< 1 channel signed 32-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_2X32 = 0x11, /**< 2 channel signed 32-bit integers */ + HIP_RES_VIEW_FORMAT_SINT_4X32 = 0x12, /**< 4 channel signed 32-bit integers */ + HIP_RES_VIEW_FORMAT_FLOAT_1X16 = 0x13, /**< 1 channel 16-bit floating point */ + HIP_RES_VIEW_FORMAT_FLOAT_2X16 = 0x14, /**< 2 channel 16-bit floating point */ + HIP_RES_VIEW_FORMAT_FLOAT_4X16 = 0x15, /**< 4 channel 16-bit floating point */ + HIP_RES_VIEW_FORMAT_FLOAT_1X32 = 0x16, /**< 1 channel 32-bit floating point */ + HIP_RES_VIEW_FORMAT_FLOAT_2X32 = 0x17, /**< 2 channel 32-bit floating point */ + HIP_RES_VIEW_FORMAT_FLOAT_4X32 = 0x18, /**< 4 channel 32-bit floating point */ + HIP_RES_VIEW_FORMAT_UNSIGNED_BC1 = 0x19, /**< Block compressed 1 */ + HIP_RES_VIEW_FORMAT_UNSIGNED_BC2 = 0x1a, /**< Block compressed 2 */ + HIP_RES_VIEW_FORMAT_UNSIGNED_BC3 = 0x1b, /**< Block compressed 3 */ + HIP_RES_VIEW_FORMAT_UNSIGNED_BC4 = 0x1c, /**< Block compressed 4 unsigned */ + HIP_RES_VIEW_FORMAT_SIGNED_BC4 = 0x1d, /**< Block compressed 4 signed */ + HIP_RES_VIEW_FORMAT_UNSIGNED_BC5 = 0x1e, /**< Block compressed 5 unsigned */ + HIP_RES_VIEW_FORMAT_SIGNED_BC5 = 0x1f, /**< Block compressed 5 signed */ + HIP_RES_VIEW_FORMAT_UNSIGNED_BC6H = 0x20, /**< Block compressed 6 unsigned half-float */ + HIP_RES_VIEW_FORMAT_SIGNED_BC6H = 0x21, /**< Block compressed 6 signed half-float */ + HIP_RES_VIEW_FORMAT_UNSIGNED_BC7 = 0x22 /**< Block compressed 7 */ +} HIPresourceViewFormat; + /** * HIP resource descriptor */ @@ -204,6 +284,39 @@ typedef struct hipResourceDesc { } res; }hipResourceDesc; +typedef struct HIP_RESOURCE_DESC_st +{ + HIPresourcetype resType; /**< Resource type */ + + union { + struct { + hipArray_t hArray; /**< HIP array */ + } array; + struct { + hipMipmappedArray_t hMipmappedArray; /**< HIP mipmapped array */ + } mipmap; + struct { + hipDeviceptr_t devPtr; /**< Device pointer */ + hipArray_Format format; /**< Array format */ + unsigned int numChannels; /**< Channels per array element */ + size_t sizeInBytes; /**< Size in bytes */ + } linear; + struct { + hipDeviceptr_t devPtr; /**< Device pointer */ + hipArray_Format format; /**< Array format */ + unsigned int numChannels; /**< Channels per array element */ + size_t width; /**< Width of the array in elements */ + size_t height; /**< Height of the array in elements */ + size_t pitchInBytes; /**< Pitch between two rows in bytes */ + } pitch2D; + struct { + int reserved[32]; + } reserved; + } res; + + unsigned int flags; /**< Flags (must be zero) */ +} HIP_RESOURCE_DESC; + /** * hip resource view descriptor */ @@ -218,6 +331,22 @@ struct hipResourceViewDesc { unsigned int lastLayer; }; +/** + * Resource view descriptor + */ +typedef struct HIP_RESOURCE_VIEW_DESC_st +{ + HIPresourceViewFormat format; /**< Resource view format */ + size_t width; /**< Width of the resource view */ + size_t height; /**< Height of the resource view */ + size_t depth; /**< Depth of the resource view */ + unsigned int firstMipmapLevel; /**< First defined mipmap level */ + unsigned int lastMipmapLevel; /**< Last defined mipmap level */ + unsigned int firstLayer; /**< First layer index */ + unsigned int lastLayer; /**< Last layer index */ + unsigned int reserved[16]; +} HIP_RESOURCE_VIEW_DESC; + /** * Memory copy types * diff --git a/include/hip/hcc_detail/hip_runtime_api.h b/include/hip/hcc_detail/hip_runtime_api.h index 2b5649e721..67bd2486d0 100644 --- a/include/hip/hcc_detail/hip_runtime_api.h +++ b/include/hip/hcc_detail/hip_runtime_api.h @@ -3491,6 +3491,27 @@ hipError_t hipMipmappedArrayGetLevel( hipArray_t* pLevelArray, hipMipmappedArray_t hMipMappedArray, unsigned int level); + +hipError_t hipTexObjectCreate( + hipTextureObject_t* pTexObject, + const HIP_RESOURCE_DESC* pResDesc, + const HIP_TEXTURE_DESC* pTexDesc, + const HIP_RESOURCE_VIEW_DESC* pResViewDesc); + +hipError_t hipTexObjectDestroy( + hipTextureObject_t texObject); + +hipError_t hipTexObjectGetResourceDesc( + HIP_RESOURCE_DESC* pResDesc, + hipTextureObject_t texObject); + +hipError_t hipTexObjectGetResourceViewDesc( + HIP_RESOURCE_VIEW_DESC* pResViewDesc, + hipTextureObject_t texObject); + +hipError_t hipTexObjectGetTextureDesc( + HIP_TEXTURE_DESC* pTexDesc, + hipTextureObject_t texObject); #endif /** diff --git a/vdi/hip_conversions.hpp b/vdi/hip_conversions.hpp index 0b8557ee4d..0515c5c56b 100644 --- a/vdi/hip_conversions.hpp +++ b/vdi/hip_conversions.hpp @@ -654,4 +654,245 @@ HIP_MEMCPY3D getDrvMemcpy3DDesc(const hipMemcpy3DParms& desc) { return descDrv; } + +inline +hipResourceType getResourceType(const HIPresourcetype resType) { + // These two enums should be isomorphic. + return static_cast(resType); +} + +inline +HIPresourcetype getResourceType(const hipResourceType resType) { + // These two enums should be isomorphic. + return static_cast(resType); +} + +inline +hipResourceDesc getResourceDesc(const HIP_RESOURCE_DESC& resDesc) { + hipResourceDesc desc; + + desc.resType = getResourceType(resDesc.resType); + switch (resDesc.resType) { + case hipResourceTypeArray: + desc.res.array.array = resDesc.res.array.hArray; + break; + case hipResourceTypeMipmappedArray: + desc.res.mipmap.mipmap = resDesc.res.mipmap.hMipmappedArray; + break; + case hipResourceTypeLinear: + desc.res.linear.devPtr = resDesc.res.linear.devPtr; + desc.res.linear.desc = getChannelFormatDesc(resDesc.res.linear.numChannels, resDesc.res.linear.format); + desc.res.linear.sizeInBytes = resDesc.res.linear.sizeInBytes; + break; + case hipResourceTypePitch2D: + desc.res.pitch2D.devPtr = resDesc.res.pitch2D.devPtr; + desc.res.pitch2D.desc = getChannelFormatDesc(resDesc.res.pitch2D.numChannels, resDesc.res.pitch2D.format); + desc.res.pitch2D.width = resDesc.res.pitch2D.width; + desc.res.pitch2D.height = resDesc.res.pitch2D.height; + desc.res.pitch2D.pitchInBytes = resDesc.res.pitch2D.pitchInBytes; + break; + default: + break; + } + + return desc; +} + +inline +HIP_RESOURCE_DESC getResourceDesc(const hipResourceDesc& resDesc) { + HIP_RESOURCE_DESC desc; + + desc.resType = getResourceType(resDesc.resType); + switch (resDesc.resType) { + case HIP_RESOURCE_TYPE_ARRAY: + desc.res.array.hArray = resDesc.res.array.array; + break; + case HIP_RESOURCE_TYPE_MIPMAPPED_ARRAY: + desc.res.mipmap.hMipmappedArray = resDesc.res.mipmap.mipmap; + break; + case HIP_RESOURCE_TYPE_LINEAR: + desc.res.linear.devPtr = resDesc.res.linear.devPtr; + desc.res.linear.numChannels = getNumChannels(resDesc.res.linear.desc); + desc.res.linear.format = getArrayFormat(resDesc.res.linear.desc); + desc.res.linear.sizeInBytes = resDesc.res.linear.sizeInBytes; + break; + case HIP_RESOURCE_TYPE_PITCH2D: + desc.res.pitch2D.devPtr = resDesc.res.pitch2D.devPtr; + desc.res.pitch2D.numChannels = getNumChannels(resDesc.res.pitch2D.desc); + desc.res.pitch2D.format = getArrayFormat(resDesc.res.pitch2D.desc); + desc.res.pitch2D.width = resDesc.res.pitch2D.width; + desc.res.pitch2D.height = resDesc.res.pitch2D.height; + desc.res.pitch2D.pitchInBytes = resDesc.res.pitch2D.pitchInBytes; + break; + default: + break; + } + + return desc; +} + +inline +hipTextureAddressMode getAddressMode(const HIPaddress_mode mode) { + // These two enums should be isomorphic. + return static_cast(mode); +} + +inline +HIPaddress_mode getAddressMode(const hipTextureAddressMode mode) { + // These two enums should be isomorphic. + return static_cast(mode); +} + +inline +hipTextureFilterMode getFilterMode(const HIPfilter_mode mode) { + // These two enums should be isomorphic. + return static_cast(mode); +} + +inline +HIPfilter_mode getFilterMode(const hipTextureFilterMode mode) { + // These two enums should be isomorphic. + return static_cast(mode); +} + +inline +hipTextureReadMode getReadMode(const unsigned int flags) { + if (flags & HIP_TRSF_READ_AS_INTEGER) { + return hipReadModeElementType; + } else { + return hipReadModeNormalizedFloat; + } +} + +inline +unsigned int getReadMode(const hipTextureReadMode mode) { + if (mode == hipReadModeElementType) { + return HIP_TRSF_READ_AS_INTEGER; + } else { + return 0; + } +} + +inline +int getsRGB(const unsigned int flags) { + if (flags & HIP_TRSF_SRGB) { + return 1; + } else { + return 0; + } +} + +inline +unsigned int getsRGB(const int sRGB) { + if (sRGB == 1) { + return HIP_TRSF_SRGB; + } else { + return 0; + } +} + +inline +int getNormalizedCoords(const unsigned int flags) { + if (flags & HIP_TRSF_NORMALIZED_COORDINATES) { + return 1; + } else { + return 0; + } +} + +inline +unsigned int getNormalizedCoords(const int normalizedCoords) { + if (normalizedCoords == 1) { + return HIP_TRSF_NORMALIZED_COORDINATES; + } else { + return 0; + } +} + +inline +hipTextureDesc getTextureDesc(const HIP_TEXTURE_DESC& texDesc) { + hipTextureDesc desc; + + desc.addressMode[0] = getAddressMode(texDesc.addressMode[0]); + desc.addressMode[1] = getAddressMode(texDesc.addressMode[1]); + desc.addressMode[2] = getAddressMode(texDesc.addressMode[2]); + desc.filterMode = getFilterMode(texDesc.filterMode); + desc.readMode = getReadMode(texDesc.flags); + desc.sRGB = getsRGB(texDesc.flags); + std::memcpy(desc.borderColor, texDesc.borderColor, sizeof(desc.borderColor)); + desc.normalizedCoords = getNormalizedCoords(texDesc.flags); + desc.maxAnisotropy = texDesc.maxAnisotropy; + desc.mipmapFilterMode = getFilterMode(texDesc.mipmapFilterMode); + desc.mipmapLevelBias = texDesc.mipmapLevelBias; + desc.minMipmapLevelClamp = texDesc.minMipmapLevelClamp; + desc.maxMipmapLevelClamp = texDesc.maxMipmapLevelClamp; + + return desc; +} + +inline +HIP_TEXTURE_DESC getTextureDesc(const hipTextureDesc& texDesc) { + HIP_TEXTURE_DESC desc; + + desc.addressMode[0] = getAddressMode(texDesc.addressMode[0]); + desc.addressMode[1] = getAddressMode(texDesc.addressMode[1]); + desc.addressMode[2] = getAddressMode(texDesc.addressMode[2]); + desc.filterMode = getFilterMode(texDesc.filterMode); + desc.flags = 0; + desc.flags |= getReadMode(texDesc.readMode); + desc.flags |= getsRGB(texDesc.sRGB); + desc.flags |= getNormalizedCoords(texDesc.normalizedCoords); + desc.maxAnisotropy = texDesc.maxAnisotropy; + desc.mipmapFilterMode = getFilterMode(texDesc.mipmapFilterMode); + desc.mipmapLevelBias = texDesc.mipmapLevelBias; + desc.minMipmapLevelClamp = texDesc.minMipmapLevelClamp; + desc.maxMipmapLevelClamp = texDesc.maxMipmapLevelClamp; + std::memcpy(desc.borderColor, texDesc.borderColor, sizeof(desc.borderColor)); + + return desc; +} + +inline +hipResourceViewFormat getResourceViewFormat(const HIPresourceViewFormat format) { + // These two enums should be isomorphic. + return static_cast(format); +} + +inline +HIPresourceViewFormat getResourceViewFormat(const hipResourceViewFormat format) { + // These two enums should be isomorphic. + return static_cast(format); +} + +inline +hipResourceViewDesc getResourceViewDesc(const HIP_RESOURCE_VIEW_DESC& resViewDesc) { + hipResourceViewDesc desc; + + desc.format = getResourceViewFormat(resViewDesc.format); + desc.width = resViewDesc.width; + desc.height = resViewDesc.height; + desc.depth = resViewDesc.depth; + desc.firstMipmapLevel = resViewDesc.firstMipmapLevel; + desc.lastMipmapLevel = resViewDesc.lastMipmapLevel; + desc.firstLayer = resViewDesc.firstLayer; + desc.lastLayer = resViewDesc.lastLayer; + + return desc; +} + +inline +HIP_RESOURCE_VIEW_DESC getResourceViewDesc(const hipResourceViewDesc& resViewDesc) { + HIP_RESOURCE_VIEW_DESC desc; + + desc.format = getResourceViewFormat(resViewDesc.format); + desc.width = resViewDesc.width; + desc.height = resViewDesc.height; + desc.depth = resViewDesc.depth; + desc.firstMipmapLevel = resViewDesc.firstMipmapLevel; + desc.lastMipmapLevel = resViewDesc.lastMipmapLevel; + desc.firstLayer = resViewDesc.firstLayer; + desc.lastLayer = resViewDesc.lastLayer; + + return desc; +} }; diff --git a/vdi/hip_hcc.def.in b/vdi/hip_hcc.def.in index 6bf962a084..d444e34147 100644 --- a/vdi/hip_hcc.def.in +++ b/vdi/hip_hcc.def.in @@ -238,3 +238,8 @@ hipFreeMipmappedArray hipMipmappedArrayGetLevel hipGetMipmappedArrayLevel hipMallocHost +hipTexObjectCreate +hipTexObjectDestroy +hipTexObjectGetResourceDesc +hipTexObjectGetResourceViewDesc +hipTexObjectGetTextureDesc diff --git a/vdi/hip_hcc.map.in b/vdi/hip_hcc.map.in index ec1c8a4558..46c59f8123 100644 --- a/vdi/hip_hcc.map.in +++ b/vdi/hip_hcc.map.in @@ -232,6 +232,11 @@ global: hipMipmappedArrayGetLevel; hipGetMipmappedArrayLevel; hipMallocHost; + hipTexObjectCreate; + hipTexObjectDestroy; + hipTexObjectGetResourceDesc; + hipTexObjectGetResourceViewDesc; + hipTexObjectGetTextureDesc; extern "C++" { hip_impl::hipLaunchKernelGGLImpl*; hip_impl::demangle*; diff --git a/vdi/hip_texture.cpp b/vdi/hip_texture.cpp index f838d6cb0c..fcea0bbb47 100644 --- a/vdi/hip_texture.cpp +++ b/vdi/hip_texture.cpp @@ -1134,4 +1134,74 @@ hipError_t hipTexRefSetMipmappedArray(textureReference* texRef, hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(mipmappedArray, format); HIP_RETURN(ihipCreateTextureObject(&texRef->textureObject, &resDesc, &texDesc, &resViewDesc)); -} \ No newline at end of file +} + +hipError_t hipTexObjectCreate(hipTextureObject_t* pTexObject, + const HIP_RESOURCE_DESC* pResDesc, + const HIP_TEXTURE_DESC* pTexDesc, + const HIP_RESOURCE_VIEW_DESC* pResViewDesc) { + HIP_INIT_API(hipTexObjectCreate, pTexObject, pResDesc, pTexDesc, pResViewDesc); + + if ((pTexObject == nullptr) || + (pResDesc == nullptr) || (pTexDesc == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + hipResourceDesc resDesc = hip::getResourceDesc(*pResDesc); + hipTextureDesc texDesc = hip::getTextureDesc(*pTexDesc); + + if (pResViewDesc != nullptr) { + hipResourceViewDesc resViewDesc = hip::getResourceViewDesc(*pResViewDesc); + HIP_RETURN(ihipCreateTextureObject(pTexObject, &resDesc, &texDesc, &resViewDesc)); + } else { + HIP_RETURN(ihipCreateTextureObject(pTexObject, &resDesc, &texDesc, nullptr)); + } +} + +hipError_t hipTexObjectDestroy(hipTextureObject_t texObject) { + HIP_INIT_API(hipTexObjectDestroy, texObject); + + HIP_RETURN(ihipDestroyTextureObject(texObject)); +} + +hipError_t hipTexObjectGetResourceDesc(HIP_RESOURCE_DESC* pResDesc, + hipTextureObject_t texObject) { + HIP_INIT_API(hipTexObjectGetResourceDesc, pResDesc, texObject); + + if ((pResDesc == nullptr) || + (texObject == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pResDesc = hip::getResourceDesc(texObject->resDesc); + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexObjectGetResourceViewDesc(HIP_RESOURCE_VIEW_DESC* pResViewDesc, + hipTextureObject_t texObject) { + HIP_INIT_API(hipTexObjectGetResourceViewDesc, pResViewDesc, texObject); + + if ((pResViewDesc == nullptr) || + (texObject == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pResViewDesc = hip::getResourceViewDesc(texObject->resViewDesc); + + HIP_RETURN(hipSuccess); +} + +hipError_t hipTexObjectGetTextureDesc(HIP_TEXTURE_DESC* pTexDesc, + hipTextureObject_t texObject) { + HIP_INIT_API(hipTexObjectGetTextureDesc, pTexDesc, texObject); + + if ((pTexDesc == nullptr) || + (texObject == nullptr)) { + HIP_RETURN(hipErrorInvalidValue); + } + + *pTexDesc = hip::getTextureDesc(texObject->texDesc); + + HIP_RETURN(hipSuccess); +} From 2c246dabfe7cf12cc57d14ef7b160cc5e57df131 Mon Sep 17 00:00:00 2001 From: Paul Fultz II Date: Wed, 18 Mar 2020 00:48:58 -0500 Subject: [PATCH 153/154] Add missing flags for hip::device target on hip-clang (#1230) This adds the missing compilation flags to hip::device so it can compile with hip-clang compiler. Change-Id: Ie2b30ea606bfca385a0e84ae03ee0a8d828ad16a --- hip-config-clang.cmake.in | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/hip-config-clang.cmake.in b/hip-config-clang.cmake.in index 9680b2ff8e..de7858a63c 100644 --- a/hip-config-clang.cmake.in +++ b/hip-config-clang.cmake.in @@ -50,9 +50,46 @@ set_and_check( hip_BIN_INSTALL_DIR "@PACKAGE_BIN_INSTALL_DIR@" ) set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc") set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig") +get_filename_component(HIP_CLANG_ROOT "${CMAKE_CXX_COMPILER}" PATH) +get_filename_component(HIP_CLANG_ROOT "${HIP_CLANG_ROOT}" PATH) +file(GLOB HIP_CLANG_INCLUDE_SEARCH_PATHS ${HIP_CLANG_ROOT}/lib/clang/*/include) +find_path(HIP_CLANG_INCLUDE_PATH stddef.h + HINTS + ${HIP_CLANG_INCLUDE_SEARCH_PATHS} + NO_DEFAULT_PATH) + find_dependency(amd_comgr) +find_dependency(AMDDeviceLibs) +set(AMDGPU_TARGETS "gfx900;gfx906" CACHE STRING "AMD GPU targets to compile for") +set(GPU_TARGETS "${AMDGPU_TARGETS}" CACHE STRING "GPU targets to compile for") + include( "${CMAKE_CURRENT_LIST_DIR}/hip-targets.cmake" ) +set_property(TARGET hip::device APPEND PROPERTY + INTERFACE_COMPILE_OPTIONS -x hip --hip-device-lib-path=${AMD_DEVICE_LIBS_PREFIX}/lib +) + +set_property(TARGET hip::device APPEND PROPERTY + INTERFACE_LINK_LIBRARIES --hip-device-lib-path=${AMD_DEVICE_LIBS_PREFIX}/lib --hip-link +) + +set_property(TARGET hip::device APPEND PROPERTY + INTERFACE_INCLUDE_DIRECTORIES "${HIP_CLANG_INCLUDE_PATH}" +) + +set_property(TARGET hip::device APPEND PROPERTY + INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${HIP_CLANG_INCLUDE_PATH}" +) + +foreach(GPU_TARGET ${GPU_TARGETS}) + set_property(TARGET hip::device APPEND PROPERTY + INTERFACE_COMPILE_OPTIONS "--cuda-gpu-arch=${GPU_TARGET}" + ) + set_property(TARGET hip::device APPEND PROPERTY + INTERFACE_LINK_LIBRARIES "--cuda-gpu-arch=${GPU_TARGET}" + ) +endforeach() + set( hip_LIBRARIES hip::host hip::device) set( hip_LIBRARY ${hip_LIBRARIES}) From 79d8f7e47e2d3c121521994002f4bf06b14eefbd Mon Sep 17 00:00:00 2001 From: Michael LIAO Date: Thu, 2 Apr 2020 12:43:08 -0400 Subject: [PATCH 154/154] [hip] Clean up unnecessary casting. Change-Id: I64b08aaef5c67ffb49330c9c605611f1fbd3f5a2 --- vdi/hip_platform.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vdi/hip_platform.cpp b/vdi/hip_platform.cpp index ca79c5300d..7e668b16cc 100755 --- a/vdi/hip_platform.cpp +++ b/vdi/hip_platform.cpp @@ -283,7 +283,7 @@ void PlatformState::registerVarSym(const void *hostVar, const char *symbolName) void PlatformState::registerVar(const char* hostvar, const DeviceVar& rvar) { amd::ScopedLock lock(lock_); - vars_.insert(std::make_pair(std::string(reinterpret_cast(hostvar)), rvar)); + vars_.insert(std::make_pair(std::string(hostvar), rvar)); } void PlatformState::registerFunction(const void* hostFunction, @@ -464,7 +464,7 @@ bool PlatformState::getTexRef(const char* hostVar, hipModule_t hmod, textureRefe bool PlatformState::getGlobalVar(const char* hostVar, int deviceId, hipModule_t hmod, hipDeviceptr_t* dev_ptr, size_t* size_ptr) { amd::ScopedLock lock(lock_); - DeviceVar* dvar = findVar(std::string(reinterpret_cast(hostVar)), deviceId, hmod); + DeviceVar* dvar = findVar(std::string(hostVar), deviceId, hmod); if (dvar != nullptr) { if (dvar->rvars[deviceId].getdeviceptr() == nullptr) { size_t sym_size = 0;