Re-sync with upstream. Add integer abs.

include/hip/hcc_detail/device_functions.h

@@ -88,6 +88,11 @@ __device__ static inline unsigned int __usad(unsigned int x, unsigned int y, uns
     return __hip_hc_ir_usad_int(x, y, z);
 }
 
+extern __device__ __attribute__((const)) unsigned int __mbcnt_lo(unsigned int x, unsigned int y) __asm("llvm.amdgcn.mbcnt.lo");
+extern __device__ __attribute__((const)) unsigned int __mbcnt_hi(unsigned int x, unsigned int y) __asm("llvm.amdgcn.mbcnt.hi");
+
+__device__ static inline unsigned int __lane_id() { return  __mbcnt_hi(-1, __mbcnt_lo(-1, 0)); }
+
 /*
 Rounding modes are not yet supported in HIP
 */

include/hip/hcc_detail/hip_runtime.h

@@ -29,6 +29,20 @@ THE SOFTWARE.
 #ifndef HIP_INCLUDE_HIP_HCC_DETAIL_HIP_RUNTIME_H
 #define HIP_INCLUDE_HIP_HCC_DETAIL_HIP_RUNTIME_H
 
+#if defined(__HCC__)
+#define __HCC_OR_HIP_CLANG__ 1
+#define __HCC_ONLY__ 1
+#define __HIP_CLANG_ONLY__ 0
+#elif defined(__clang__) && defined(__HIP__)
+#define __HCC_OR_HIP_CLANG__ 1
+#define __HCC_ONLY__ 0
+#define __HIP_CLANG_ONLY__ 1
+#else
+#define __HCC_OR_HIP_CLANG__ 0
+#define __HCC_ONLY__ 0
+#define __HIP_CLANG_ONLY__ 0
+#endif
+
 //---
 // Top part of file can be compiled with any compiler
 
@@ -41,15 +55,16 @@ THE SOFTWARE.
 #include <stddef.h>
 #endif  //__cplusplus
 
-#if __HCC__
+#if __HCC_OR_HIP_CLANG__
 
 // Define NVCC_COMPAT for CUDA compatibility
 #define NVCC_COMPAT
 #define CUDA_SUCCESS hipSuccess
 
 #include <hip/hip_runtime_api.h>
+#endif  // __HCC_OR_HIP_CLANG__
 
-
+#if __HCC__
 // define HIP_ENABLE_PRINTF to enable printf
 #ifdef HIP_ENABLE_PRINTF
 #define HCC_ENABLE_ACCELERATOR_PRINTF 1
@@ -164,6 +179,10 @@ extern int HIP_TRACE_API;
 #define __HCC_C__
 #endif
 
+#endif  // defined __HCC__
+
+#if __HCC_OR_HIP_CLANG__
+
 // TODO - hipify-clang - change to use the function call.
 //#define warpSize hc::__wavesize()
 static constexpr int warpSize = 64;
@@ -371,6 +390,10 @@ __device__ void __threadfence_system(void);
  * @}
  */
 
+#endif  // __HCC_OR_HIP_CLANG__
+
+#if defined __HCC__
+
 template <
     typename std::common_type<decltype(hc_get_group_id), decltype(hc_get_group_size),
                               decltype(hc_get_num_groups), decltype(hc_get_workitem_id)>::type f>
@@ -414,6 +437,8 @@ static constexpr Coordinates<hc_get_workitem_id> threadIdx;
 #define hipGridDim_y (hc_get_num_groups(1))
 #define hipGridDim_z (hc_get_num_groups(2))
 
+#endif // defined __HCC__
+#if __HCC_OR_HIP_CLANG__
 extern "C" __device__ void* __hip_hc_memcpy(void* dst, const void* src, size_t size);
 extern "C" __device__ void* __hip_hc_memset(void* ptr, uint8_t val, size_t size);
 extern "C" __device__ void* __hip_hc_malloc(size_t);
@@ -446,7 +471,9 @@ static inline __device__ void printf(const char* format, All... all) {}
 #endif
 
 #endif
+#endif //__HCC_OR_HIP_CLANG__
 
+#ifdef __HCC__
 
 #define __syncthreads() hc_barrier(CLK_LOCAL_MEM_FENCE)
 
@@ -514,7 +541,9 @@ extern void ihipPostLaunchKernel(const char* kernelName, hipStream_t stream, gri
  *   @}
  */
 
-
+//
+// hip-clang functions
+//
 #elif defined(__clang__) && defined(__HIP__)
 
 #define HIP_KERNEL_NAME(...) __VA_ARGS__
@@ -612,6 +641,27 @@ extern const __device__ __attribute__((weak)) __hip_builtin_gridDim_t gridDim;
 #define hipGridDim_y gridDim.y
 #define hipGridDim_z gridDim.z
 
+#pragma push_macro("__DEVICE__")
+#define __DEVICE__ extern "C" __device__ __attribute__((always_inline)) \
+  __attribute__((weak))
+
+__DEVICE__ void __device_trap() __asm("llvm.trap");
+
+__DEVICE__ void inline __assert_fail(const char * __assertion,
+                                     const char *__file,
+                                     unsigned int __line,
+                                     const char *__function)
+{
+    // Ignore all the args for now.
+    __device_trap();
+}
+
+extern "C" __device__ __attribute__((noduplicate)) void __syncthreads();
+
+#pragma push_macro("__DEVICE__")
+
+#include <hip/hcc_detail/math_functions.h>
+
 #endif
 
 #endif  // HIP_HCC_DETAIL_RUNTIME_H

include/hip/hcc_detail/hip_runtime_api.h

@@ -94,6 +94,19 @@ typedef struct ihipModule_t* hipModule_t;
 
 typedef struct ihipModuleSymbol_t* hipFunction_t;
 
+struct hipFuncAttributes {
+    int binaryVersion;
+    int cacheModeCA;
+    size_t constSizeBytes;
+    size_t localSizeBytes;
+    int maxDynamicSharedSizeBytes;
+    int maxThreadsPerBlock;
+    int numRegs;
+    int preferredShmemCarveout;
+    int ptxVersion;
+    size_t sharedSizeBytes;
+};
+
 typedef struct ihipEvent_t* hipEvent_t;
 
 enum hipLimit_t {
@@ -1450,6 +1463,27 @@ hipError_t hipMemset2D(void* dst, size_t pitch, int value, size_t width, size_t
 
 hipError_t hipMemset2DAsync(void* dst, size_t pitch, int value, size_t width, size_t height,hipStream_t stream __dparm(0));
 
+/**
+ *  @brief Fills synchronously the memory area pointed to by pitchedDevPtr with the constant value.
+ *
+ *  @param[in] pitchedDevPtr
+ *  @param[in]  value - constant value to be set
+ *  @param[in]  extent
+ *  @return #hipSuccess, #hipErrorInvalidValue, #hipErrorMemoryFree
+ */
+hipError_t hipMemset3D(hipPitchedPtr pitchedDevPtr, int  value, hipExtent extent );
+
+/**
+ *  @brief Fills asynchronously the memory area pointed to by pitchedDevPtr with the constant value.
+ *
+ *  @param[in] pitchedDevPtr
+ *  @param[in]  value - constant value to be set
+ *  @param[in]  extent
+ *  @param[in]  stream
+ *  @return #hipSuccess, #hipErrorInvalidValue, #hipErrorMemoryFree
+ */
+hipError_t hipMemset3DAsync(hipPitchedPtr pitchedDevPtr, int  value, hipExtent extent ,hipStream_t stream __dparm(0));
+
 /**
  * @brief Query memory info.
  * Return snapshot of free memory, and total allocatable memory on the device.
@@ -2222,6 +2256,17 @@ hipError_t hipModuleUnload(hipModule_t module);
  */
 hipError_t hipModuleGetFunction(hipFunction_t* function, hipModule_t module, const char* kname);
 
+/**
+ * @bried Find out attributes for a given function.
+ *
+ * @param [out] attr
+ * @param [in] func
+ *
+ * @returns hipSuccess, hipErrorInvalidDeviceFunction
+ */
+
+hipError_t hipFuncGetAttributes(hipFuncAttributes* attr, const void* func);
+
 /**
  * @brief returns device memory pointer and size of the kernel present in the module with symbol @p
  * name

include/hip/hcc_detail/math_functions.h

@@ -27,6 +27,7 @@ THE SOFTWARE.
 #include <hip/hip_runtime.h>
 
 #include <assert.h>
+#include <limits.h>
 #include <stdint.h>
 
 __device__
@@ -101,6 +102,9 @@ uint64_t __make_mantissa(const char* tagp)
 // BEGIN FLOAT
 __device__
 inline
+float abs(float x) { return __ocml_fabs_f32(x); }
+__device__
+inline
 float acosf(float x) { return __ocml_acos_f32(x); }
 __device__
 inline
@@ -628,6 +632,9 @@ float __tanf(float x) { return __ocml_tan_f32(x); }
 // BEGIN DOUBLE
 __device__
 inline
+double abs(double x) { return __ocml_fabs_f64(x); }
+__device__
+inline
 double acos(double x) { return __ocml_acos_f64(x); }
 __device__
 inline
@@ -1101,4 +1108,37 @@ double __fma_rz(double x, double y, double z)
     return __llvm_fma_rtz_f64(x, y, z);
 }
 // END INTRINSICS
-// END DOUBLE
+// END DOUBLE
+
+// BEGIN INTEGER
+__device__
+inline
+int abs(int x)
+{
+    int sgn = x >> (sizeof(int) * CHAR_BIT - 1);
+    return (x ^ sgn) - sgn;
+}
+__device__
+inline
+long labs(long x)
+{
+    long sgn = x >> (sizeof(long) * CHAR_BIT - 1);
+    return (x ^ sgn) - sgn;
+}
+__device__
+inline
+long long llabs(long long x)
+{
+    long long sgn = x >> (sizeof(long long) * CHAR_BIT - 1);
+    return (x ^ sgn) - sgn;
+}
+
+#if defined(__cplusplus)
+    __device__
+    inline
+    long abs(long x) { return labs(x); }
+    __device__
+    inline
+    long long abs(long long x) { return llabs(x); }
+#endif
+// END INTEGER

include/hip/hcc_detail/program_state.hpp

@@ -22,8 +22,10 @@ THE SOFTWARE.
 
 #pragma once
 
+#include <hsa/amd_hsa_kernel_code.h>
 #include <hsa/hsa.h>
 #include <hsa/hsa_ext_amd.h>
+#include <hsa/hsa_ven_amd_loader.h>
 
 #include <cstddef>
 #include <istream>
@@ -46,11 +48,45 @@ struct hash<hsa_agent_t> {
 inline constexpr bool operator==(hsa_agent_t x, hsa_agent_t y) { return x.handle == y.handle; }
 
 namespace hip_impl {
-struct Kernel_descriptor {
-    std::uint64_t kernel_object_;
-    std::uint32_t group_size_;
-    std::uint32_t private_size_;
-    std::string name_;
+class Kernel_descriptor {
+    std::uint64_t kernel_object_{};
+    amd_kernel_code_t const* kernel_header_{nullptr};
+    std::string name_{};
+public:
+    Kernel_descriptor() = default;
+    Kernel_descriptor(std::uint64_t kernel_object, const std::string& name)
+        : kernel_object_{kernel_object}, name_{name}
+    {
+        bool supported{false};
+        std::uint16_t min_v{UINT16_MAX};
+        auto r = hsa_system_major_extension_supported(
+            HSA_EXTENSION_AMD_LOADER, 1, &min_v, &supported);
+
+        if (r != HSA_STATUS_SUCCESS || !supported) return;
+
+        hsa_ven_amd_loader_1_01_pfn_t tbl{};
+
+        r = hsa_system_get_major_extension_table(
+            HSA_EXTENSION_AMD_LOADER,
+            1,
+            sizeof(tbl),
+            reinterpret_cast<void*>(&tbl));
+
+        if (r != HSA_STATUS_SUCCESS) return;
+        if (!tbl.hsa_ven_amd_loader_query_host_address) return;
+
+        r = tbl.hsa_ven_amd_loader_query_host_address(
+            reinterpret_cast<void*>(kernel_object_),
+            reinterpret_cast<const void**>(&kernel_header_));
+
+        if (r != HSA_STATUS_SUCCESS) return;
+    }
+    Kernel_descriptor(const Kernel_descriptor&) = default;
+    Kernel_descriptor(Kernel_descriptor&&) = default;
+    ~Kernel_descriptor() = default;
+
+    Kernel_descriptor& operator=(const Kernel_descriptor&) = default;
+    Kernel_descriptor& operator=(Kernel_descriptor&&) = default;
 
     operator hipFunction_t() const {  // TODO: this is awful and only meant for illustration.
         return reinterpret_cast<hipFunction_t>(const_cast<Kernel_descriptor*>(this));

include/hip/hip_runtime_api.h

@@ -243,6 +243,8 @@ typedef enum __HIP_NODISCARD hipError_t {
         1062,  ///< Produced when trying to unlock a non-page-locked memory.
     hipErrorMapBufferObjectFailed =
         1071,    ///< Produced when the IPC memory attach failed from ROCr.
+    hipErrorAssert =
+        1081,    ///< Produced when the kernel calls assert.
     hipErrorTbd  ///< Marker that more error codes are needed.
 } hipError_t;
 

include/hip/nvcc_detail/hip_runtime_api.h

@@ -150,16 +150,20 @@ typedef CUfunction hipFunction_t;
 typedef CUdeviceptr hipDeviceptr_t;
 typedef struct cudaArray hipArray;
 typedef struct cudaArray* hipArray_const_t;
+typedef cudaFuncAttributes hipFuncAttributes;
 #define hipMemcpy3DParms cudaMemcpy3DParms
 #define hipArrayDefault cudaArrayDefault
 
 typedef cudaTextureObject_t hipTextureObject_t;
 typedef cudaSurfaceObject_t hipSurfaceObject_t;
+#define hipTextureType1D cudaTextureType1D
+#define hipTextureType1DLayered cudaTextureType1DLayered
 #define hipTextureType2D cudaTextureType2D
 #define hipTextureType3D cudaTextureType3D
 #define hipDeviceMapHost cudaDeviceMapHost
 
 #define hipExtent cudaExtent
+#define hipPitchedPtr cudaPitchedPtr
 #define make_hipExtent make_cudaExtent
 #define make_hipPos make_cudaPos
 #define make_hipPitchedPtr make_cudaPitchedPtr
@@ -381,6 +385,10 @@ inline static hipError_t hipMallocPitch(void** ptr, size_t* pitch, size_t width,
     return hipCUDAErrorTohipError(cudaMallocPitch(ptr, pitch, width, height));
 }
 
+inline static hipError_t hipMalloc3D(hipPitchedPtr* pitchedDevPtr, hipExtent extent) {
+    return hipCUDAErrorTohipError(cudaMalloc3D(pitchedDevPtr, extent));
+}
+
 inline static hipError_t hipFree(void* ptr) { return hipCUDAErrorTohipError(cudaFree(ptr)); }
 
 inline static hipError_t hipMallocHost(void** ptr, size_t size)
@@ -649,6 +657,14 @@ inline static hipError_t hipMemset2DAsync(void* dst, size_t pitch, int value, si
     return hipCUDAErrorTohipError(cudaMemset2DAsync(dst, pitch, value, width, height, stream));
 }
 
+inline static hipError_t hipMemset3D(hipPitchedPtr pitchedDevPtr, int  value, hipExtent extent ){
+    return hipCUDAErrorTohipError(cudaMemset3D(pitchedDevPtr, value, extent));
+}
+
+inline static hipError_t hipMemset3DAsync(hipPitchedPtr pitchedDevPtr, int  value, hipExtent extent, hipStream_t stream __dparm(0) ){
+    return hipCUDAErrorTohipError(cudaMemset3DAsync(pitchedDevPtr, value, extent, stream));
+}
+
 inline static hipError_t hipGetDeviceProperties(hipDeviceProp_t* p_prop, int device) {
     struct cudaDeviceProp cdprop;
     cudaError_t cerror;
@@ -1092,6 +1108,10 @@ inline static hipError_t hipModuleGetFunction(hipFunction_t* function, hipModule
     return hipCUResultTohipError(cuModuleGetFunction(function, module, kname));
 }
 
+inline static hipError_t hipFuncGetAttributes(hipFuncAttributes* attr, const void* func) {
+    return hipCUDAErrorTohipError(cudaFuncGetAttributes(attr, func));
+}
+
 inline static hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes, hipModule_t hmod,
                                             const char* name) {
     return hipCUResultTohipError(cuModuleGetGlobal(dptr, bytes, hmod, name));
@@ -1148,8 +1168,8 @@ inline static hipError_t hipBindTexture(size_t* offset, const struct texture<T,
 }
 
 template <class T, int dim, enum cudaTextureReadMode readMode>
-inline static hipError_t hipBindTexture(size_t* offset, struct texture<T, dim, readMode>* tex,
-                                        const void* devPtr, const struct hipChannelFormatDesc* desc,
+inline static hipError_t hipBindTexture(size_t* offset, struct texture<T, dim, readMode>& tex,
+                                        const void* devPtr, const struct hipChannelFormatDesc& desc,
                                         size_t size = UINT_MAX) {
     return hipCUDAErrorTohipError(cudaBindTexture(offset, tex, devPtr, desc, size));
 }
@@ -1159,6 +1179,11 @@ inline static hipError_t hipUnbindTexture(struct texture<T, dim, readMode>* tex)
     return hipCUDAErrorTohipError(cudaUnbindTexture(tex));
 }
 
+inline static hipError_t hipBindTexture(size_t* offset, textureReference* tex, const void* devPtr,
+                                        const hipChannelFormatDesc* desc, size_t size = UINT_MAX){
+    return hipCUDAErrorTohipError(cudaBindTexture(offset, tex, devPtr, desc, size));
+}
+
 template <class T, int dim, enum hipTextureReadMode readMode>
 inline static hipError_t hipBindTextureToArray(struct texture<T, dim, readMode>& tex,
                                                hipArray_const_t array,